datafolder 1.0.6 → 1.0.8

package/scripts/airbnb_queries.js

@@ -1,265 +1,214 @@
-import { deepEqual } from "node:assert/strict";
-
-function query1(qyDB) {
-    console.time("query1");
-    deepEqual(
-        qyDB
-            .queryFiles("airbnb1", {
-                property_type: "Loft",
-                room_type: "Entire home/apt",
-                bed_type: "Real Bed",
-                address: { street: "Porto, Porto, Portugal" },
-            })
-            .map((x) => x.prop("name")),
-        [
-            "Porto city centre apartment H",
-            "Romantic loft near historic center",
-            "Lovely sun light  in a stylish loft, Downtown Spot",
-            "Charming & Sunny Mezzanine Loft",
-            "Be Happy in Porto",
-            "Quarry Apartment",
-            "Loft with balcony and view of Casa da Música PORTO",
-            "For 4 or 6 - Wonderful Apartment downtown",
-            "Clérigos Apartments - Apt. 4E for 2/3 Pax",
-            "Bright Top Floor Two Rooms Apartment w/ Terrace",
-            "ReUrban Flats - Santa Catarina #3",
-            "The Blue and White Porto House - Vintage",
-            "4twoLoft at Great Location",
-            "Step In Porto -  Loft w/Mezzanine",
-            "Your OWM Invicta",
-            "Pury Apartments",
-            "Live Porto & Douro - Breiner III",
-        ],
-    );
-    console.timeEnd("query1");
-}
-function query2(qyDB) {
-    console.time("query2");
-    deepEqual(
-        qyDB
-            .queryFiles(
-                "airbnb1",
-                {
-                    property_type: "Loft",
-                    room_type: "Entire home/apt",
-                    bed_type: "Real Bed",
-                    address: { street: "Porto, Porto, Portugal" },
-                },
-                9,
-            )
-            .map((x) => x.prop("name")),
-        [
-            "Porto city centre apartment H",
-            "Romantic loft near historic center",
-            "Lovely sun light  in a stylish loft, Downtown Spot",
-            "Charming & Sunny Mezzanine Loft",
-            "Be Happy in Porto",
-            "Quarry Apartment",
-            "Loft with balcony and view of Casa da Música PORTO",
-            "For 4 or 6 - Wonderful Apartment downtown",
-            "Clérigos Apartments - Apt. 4E for 2/3 Pax",
-        ],
-    );
-    console.timeEnd("query2");
-}
-function query3(qyDB) {
-    console.time("query3");
-    deepEqual(
-        qyDB
-            .queryFiles(
-                "airbnb1",
-                {
-                    property_type: "Loft",
-                    room_type: "Entire home/apt",
-                    bed_type: "Real Bed",
-                    address: { street: "Porto, Porto, Portugal" },
-                },
-                9,
-                9,
-            )
-            .map((x) => x.prop("name")),
-        [
-            "Bright Top Floor Two Rooms Apartment w/ Terrace",
-            "ReUrban Flats - Santa Catarina #3",
-            "The Blue and White Porto House - Vintage",
-            "4twoLoft at Great Location",
-            "Step In Porto -  Loft w/Mezzanine",
-            "Your OWM Invicta",
-            "Pury Apartments",
-            "Live Porto & Douro - Breiner III",
-        ],
-    );
-    console.timeEnd("query3");
-}
-function query4(qyDB) {
-    console.time("query4");
-    deepEqual(
-        qyDB
-            .queryFiles("airbnb1", {
-                $or: [
-                    {
-                        property_type: "Loft",
-                        room_type: "Entire home/apt",
-                        bed_type: "Real Bed",
-                        address: { street: "Porto, Porto, Portugal" },
-                    },
-                    { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
-                ],
-            })
-            .map((x) => x.prop("name"))
-            .sort(),
-        [
-            "Porto city centre apartment H",
-            "Romantic loft near historic center",
-            "Lovely sun light  in a stylish loft, Downtown Spot",
-            "Charming & Sunny Mezzanine Loft",
-            "Brand New Luxury Downtown Studio",
-            "Be Happy in Porto",
-            "Quarry Apartment",
-            "Loft with balcony and view of Casa da Música PORTO",
-            "For 4 or 6 - Wonderful Apartment downtown",
-            "Clérigos Apartments - Apt. 4E for 2/3 Pax",
-            "Bright Top Floor Two Rooms Apartment w/ Terrace",
-            "ReUrban Flats - Santa Catarina #3",
-            "The Blue and White Porto House - Vintage",
-            "4twoLoft at Great Location",
-            "Step In Porto -  Loft w/Mezzanine",
-            "Your OWM Invicta",
-            "Pury Apartments",
-            "Live Porto & Douro - Breiner III",
-        ].sort(),
-    );
-    console.timeEnd("query4");
-}
-function query5(qyDB) {
-    console.time("query5");
-    deepEqual(
-        qyDB
-            .queryFiles(
-                "airbnb1",
-                {
-                    $or: [
-                        {
-                            property_type: "Loft",
-                            room_type: "Entire home/apt",
-                            bed_type: "Real Bed",
-                            address: { street: "Porto, Porto, Portugal" },
-                        },
-                        { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
-                    ],
-                },
-                3,
-            )
-            .map((x) => x.prop("name")),
-        [
-            "Porto city centre apartment H",
-            "Romantic loft near historic center",
-            "Lovely sun light  in a stylish loft, Downtown Spot",
-        ],
-    );
-    console.timeEnd("query5");
-}
-function query6(qyDB) {
-    console.time("query6");
-    deepEqual(
-        qyDB
-            .queryFiles(
-                "airbnb1",
-                {
-                    $or: [
-                        {
-                            property_type: "Loft",
-                            room_type: "Entire home/apt",
-                            bed_type: "Real Bed",
-                            address: { street: "Porto, Porto, Portugal" },
-                        },
-                        { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
-                    ],
-                },
-                20,
-                10,
-            )
-            .map((x) => x.prop("name")),
-        [
-            "ReUrban Flats - Santa Catarina #3",
-            "The Blue and White Porto House - Vintage",
-            "4twoLoft at Great Location",
-            "Step In Porto -  Loft w/Mezzanine",
-            "Your OWM Invicta",
-            "Pury Apartments",
-            "Live Porto & Douro - Breiner III",
-            "Brand New Luxury Downtown Studio",
-        ],
-    );
-    console.timeEnd("query6");
-}
-function query7(qyDB) {
-    console.time("query7");
-    deepEqual(
-        qyDB
-            .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 })
-            .map((x) => x.prop("name")),
-        ["Romantic loft near historic center", "Brand New Luxury Downtown Studio"],
-    );
-    console.timeEnd("query7");
-}
-function query8(qyDB) {
-    console.time("query8");
-    deepEqual(
-        qyDB
-            .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 }, 10, 10)
-            .map((x) => x.prop("name")),
-        [],
-    );
-    console.timeEnd("query8");
-}
-function query9(qyDB) {
-    console.time("query9");
-    deepEqual(
-        qyDB
-            .queryFiles(
-                "airbnb1",
-                {
-                    $or: [
-                        {
-                            property_type: "Loft",
-                            room_type: "Entire home/apt",
-                            bed_type: "Real Bed",
-                            address: { street: "Porto, Porto, Portugal" },
-                        },
-                        { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
-                    ],
-                },
-                3,
-            )
-            .map((x) => x.prop("name")),
-        [
-            "Porto city centre apartment H",
-            "Romantic loft near historic center",
-            "Lovely sun light  in a stylish loft, Downtown Spot",
-        ],
-    );
-    console.timeEnd("query9");
-}
-function runQueries(qyDB) {
-    query1(qyDB);
-    query1(qyDB);
-    query2(qyDB);
-    query2(qyDB);
-    query3(qyDB);
-    query3(qyDB);
-    query4(qyDB);
-    query4(qyDB);
-    query5(qyDB);
-    query5(qyDB);
-    query6(qyDB);
-    query6(qyDB);
-    query7(qyDB);
-    query7(qyDB);
-    query8(qyDB);
-    query8(qyDB);
-    query9(qyDB);
-    query9(qyDB);
-}
-
-export { runQueries };
+import { deepEqual } from "node:assert/strict";
+function query1(qyDB) {
+    console.time("query1");
+    deepEqual(qyDB
+        .queryFiles("airbnb1", {
+        property_type: "Loft",
+        room_type: "Entire home/apt",
+        bed_type: "Real Bed",
+        address: { street: "Porto, Porto, Portugal" },
+    })
+        .map((x) => x.prop("name")), [
+        "Porto city centre apartment H",
+        "Romantic loft near historic center",
+        "Lovely sun light  in a stylish loft, Downtown Spot",
+        "Charming & Sunny Mezzanine Loft",
+        "Be Happy in Porto",
+        "Quarry Apartment",
+        "Loft with balcony and view of Casa da Música PORTO",
+        "For 4 or 6 - Wonderful Apartment downtown",
+        "Clérigos Apartments - Apt. 4E for 2/3 Pax",
+        "Bright Top Floor Two Rooms Apartment w/ Terrace",
+        "ReUrban Flats - Santa Catarina #3",
+        "The Blue and White Porto House - Vintage",
+        "4twoLoft at Great Location",
+        "Step In Porto -  Loft w/Mezzanine",
+        "Your OWM Invicta",
+        "Pury Apartments",
+        "Live Porto & Douro - Breiner III",
+    ]);
+    console.timeEnd("query1");
+}
+function query2(qyDB) {
+    console.time("query2");
+    deepEqual(qyDB
+        .queryFiles("airbnb1", {
+        property_type: "Loft",
+        room_type: "Entire home/apt",
+        bed_type: "Real Bed",
+        address: { street: "Porto, Porto, Portugal" },
+    }, 9)
+        .map((x) => x.prop("name")), [
+        "Porto city centre apartment H",
+        "Romantic loft near historic center",
+        "Lovely sun light  in a stylish loft, Downtown Spot",
+        "Charming & Sunny Mezzanine Loft",
+        "Be Happy in Porto",
+        "Quarry Apartment",
+        "Loft with balcony and view of Casa da Música PORTO",
+        "For 4 or 6 - Wonderful Apartment downtown",
+        "Clérigos Apartments - Apt. 4E for 2/3 Pax",
+    ]);
+    console.timeEnd("query2");
+}
+function query3(qyDB) {
+    console.time("query3");
+    deepEqual(qyDB
+        .queryFiles("airbnb1", {
+        property_type: "Loft",
+        room_type: "Entire home/apt",
+        bed_type: "Real Bed",
+        address: { street: "Porto, Porto, Portugal" },
+    }, 9, 9)
+        .map((x) => x.prop("name")), [
+        "Bright Top Floor Two Rooms Apartment w/ Terrace",
+        "ReUrban Flats - Santa Catarina #3",
+        "The Blue and White Porto House - Vintage",
+        "4twoLoft at Great Location",
+        "Step In Porto -  Loft w/Mezzanine",
+        "Your OWM Invicta",
+        "Pury Apartments",
+        "Live Porto & Douro - Breiner III",
+    ]);
+    console.timeEnd("query3");
+}
+function query4(qyDB) {
+    console.time("query4");
+    deepEqual(qyDB
+        .queryFiles("airbnb1", {
+        $or: [
+            {
+                property_type: "Loft",
+                room_type: "Entire home/apt",
+                bed_type: "Real Bed",
+                address: { street: "Porto, Porto, Portugal" },
+            },
+            { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
+        ],
+    })
+        .map((x) => x.prop("name"))
+        .sort(), [
+        "Porto city centre apartment H",
+        "Romantic loft near historic center",
+        "Lovely sun light  in a stylish loft, Downtown Spot",
+        "Charming & Sunny Mezzanine Loft",
+        "Brand New Luxury Downtown Studio",
+        "Be Happy in Porto",
+        "Quarry Apartment",
+        "Loft with balcony and view of Casa da Música PORTO",
+        "For 4 or 6 - Wonderful Apartment downtown",
+        "Clérigos Apartments - Apt. 4E for 2/3 Pax",
+        "Bright Top Floor Two Rooms Apartment w/ Terrace",
+        "ReUrban Flats - Santa Catarina #3",
+        "The Blue and White Porto House - Vintage",
+        "4twoLoft at Great Location",
+        "Step In Porto -  Loft w/Mezzanine",
+        "Your OWM Invicta",
+        "Pury Apartments",
+        "Live Porto & Douro - Breiner III",
+    ].sort());
+    console.timeEnd("query4");
+}
+function query5(qyDB) {
+    console.time("query5");
+    deepEqual(qyDB
+        .queryFiles("airbnb1", {
+        $or: [
+            {
+                property_type: "Loft",
+                room_type: "Entire home/apt",
+                bed_type: "Real Bed",
+                address: { street: "Porto, Porto, Portugal" },
+            },
+            { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
+        ],
+    }, 3)
+        .map((x) => x.prop("name")), [
+        "Porto city centre apartment H",
+        "Romantic loft near historic center",
+        "Lovely sun light  in a stylish loft, Downtown Spot",
+    ]);
+    console.timeEnd("query5");
+}
+function query6(qyDB) {
+    console.time("query6");
+    deepEqual(qyDB
+        .queryFiles("airbnb1", {
+        $or: [
+            {
+                property_type: "Loft",
+                room_type: "Entire home/apt",
+                bed_type: "Real Bed",
+                address: { street: "Porto, Porto, Portugal" },
+            },
+            { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
+        ],
+    }, 20, 10)
+        .map((x) => x.prop("name")), [
+        "ReUrban Flats - Santa Catarina #3",
+        "The Blue and White Porto House - Vintage",
+        "4twoLoft at Great Location",
+        "Step In Porto -  Loft w/Mezzanine",
+        "Your OWM Invicta",
+        "Pury Apartments",
+        "Live Porto & Douro - Breiner III",
+        "Brand New Luxury Downtown Studio",
+    ]);
+    console.timeEnd("query6");
+}
+function query7(qyDB) {
+    console.time("query7");
+    deepEqual(qyDB
+        .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 })
+        .map((x) => x.prop("name")), ["Romantic loft near historic center", "Brand New Luxury Downtown Studio"]);
+    console.timeEnd("query7");
+}
+function query8(qyDB) {
+    console.time("query8");
+    deepEqual(qyDB
+        .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 }, 10, 10)
+        .map((x) => x.prop("name")), []);
+    console.timeEnd("query8");
+}
+function query9(qyDB) {
+    console.time("query9");
+    deepEqual(qyDB
+        .queryFiles("airbnb1", {
+        $or: [
+            {
+                property_type: "Loft",
+                room_type: "Entire home/apt",
+                bed_type: "Real Bed",
+                address: { street: "Porto, Porto, Portugal" },
+            },
+            { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
+        ],
+    }, 3)
+        .map((x) => x.prop("name")), [
+        "Porto city centre apartment H",
+        "Romantic loft near historic center",
+        "Lovely sun light  in a stylish loft, Downtown Spot",
+    ]);
+    console.timeEnd("query9");
+}
+function runQueries(qyDB) {
+    query1(qyDB);
+    query1(qyDB);
+    query2(qyDB);
+    query2(qyDB);
+    query3(qyDB);
+    query3(qyDB);
+    query4(qyDB);
+    query4(qyDB);
+    query5(qyDB);
+    query5(qyDB);
+    query6(qyDB);
+    query6(qyDB);
+    query7(qyDB);
+    query7(qyDB);
+    query8(qyDB);
+    query8(qyDB);
+    query9(qyDB);
+    query9(qyDB);
+}
+export { runQueries };

package/scripts/convertBinFiles.js

@@ -0,0 +1,47 @@
+import { join, basename } from "node:path";
+import { readFile, readdir, writeFile } from "node:fs/promises";
+import { iterPMBuffer, loadPrefixPMFile } from "../src/QySnapshots.js";
+import { ensureDir, folderOrFileExists } from "../src/QyUtils.js";
+logger.level = "info";
+run();
+async function loadPosMapFile(posMapFilePath) {
+    const buffer = await readFile(posMapFilePath);
+    const posMap = {};
+    iterPMBuffer(buffer, (key, nums) => (posMap[key] = nums));
+    return posMap;
+}
+async function run() {
+    const folderPath = process.argv[2];
+    let destFolderPath = process.argv[3];
+    if (!folderPath || !destFolderPath) {
+        console.log(`Usage: node ${basename(import.meta.filename)} folder/to/convert folder/to/save or inplace`);
+    }
+    else {
+        if (destFolderPath == "inplace")
+            destFolderPath = folderPath;
+        await convertFolder(folderPath, destFolderPath);
+    }
+}
+async function convertFolder(folder, destFolder) {
+    const promises = [];
+    for (const item of await readdir(folder, { withFileTypes: true })) {
+        const { name } = item;
+        if (item.isDirectory()) {
+            promises.push(convertFolder(join(folder, name), join(destFolder, name)));
+        }
+        else {
+            const match = /^[0-9]+_(pM|prefixPM)\.txt/.exec(name);
+            if (match) {
+                const filePath = join(folder, name);
+                const destFilePath = join(destFolder, "dump_" + name + ".json");
+                if (await folderOrFileExists(destFilePath))
+                    continue;
+                await ensureDir(destFolder);
+                console.log(`Converting ${filePath} -> ${destFilePath}`);
+                const map = await (match[1] == "pM" ? loadPosMapFile(filePath) : loadPrefixPMFile(filePath));
+                promises.push(writeFile(destFilePath, JSON.stringify(map, null, 2)));
+            }
+        }
+    }
+    await Promise.all(promises);
+}

package/scripts/downloadFile.js

@@ -1,40 +1,41 @@
-import { createWriteStream } from "fs";
-import { get } from "https";
-import { ensureParentDir, folderOrFileExists } from "../src/QyUtils.js";
-import { logger } from "../src/QyLogger.js";
-
-async function downloadFile(fileUrl, filePath) {
-  if (await folderOrFileExists(filePath)) return;
-  await ensureParentDir(filePath);
-  let resolve, reject;
-  const promise = new Promise((res, rej) => {
-    resolve = res;
-    reject = rej;
-  });
-  get(fileUrl, (response) => {
-    if (response.statusCode !== 200) {
-      logger.error(`Download failed: Status Code ${response.statusCode}`);
-      reject(response.statusCode);
-    } else {
-      const fileStream = createWriteStream(filePath);
-      let accu = 0;
-      response.on("data", (chunk) => {
-        fileStream.write(chunk);
-        accu += chunk.length;
-        if (accu >= 1000_000) {
-          process.stdout.write(".");
-          accu = 0;
-        }
-      });
-      response.on("end", () => {
-        fileStream.close();
-        resolve();
-      });
-    }
-  }).on("error", (err) => {
-    logger.error(`Request error: ${err.message}`);
-    reject(err.message);
-  });
-  return promise;
-}
-export { downloadFile };
+import { createWriteStream } from "fs";
+import { get } from "https";
+import { ensureParentDir, folderOrFileExists } from "../src/QyUtils.js";
+import { logger } from "../src/QyLogger.js";
+async function downloadFile(fileUrl, filePath) {
+    if (await folderOrFileExists(filePath))
+        return;
+    await ensureParentDir(filePath);
+    let resolve, reject;
+    const promise = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    get(fileUrl, (response) => {
+        if (response.statusCode !== 200) {
+            logger.error(`Download failed: Status Code ${response.statusCode}`);
+            reject(response.statusCode);
+        }
+        else {
+            const fileStream = createWriteStream(filePath);
+            let accu = 0;
+            response.on("data", (chunk) => {
+                fileStream.write(chunk);
+                accu += chunk.length;
+                if (accu >= 1000_000) {
+                    process.stdout.write(".");
+                    accu = 0;
+                }
+            });
+            response.on("end", () => {
+                fileStream.close();
+                resolve();
+            });
+        }
+    }).on("error", (err) => {
+        logger.error(`Request error: ${err.message}`);
+        reject(err.message);
+    });
+    return promise;
+}
+export { downloadFile };