datafolder 1.0.0

package/scripts/airbnb_queries.js

@@ -0,0 +1,264 @@
+const assert = require("node:assert/strict");
+
+function query1(qyDB) {
+    console.time("query1");
+    assert.deepEqual(
+        qyDB
+            .queryFiles("airbnb1", {
+                property_type: "Loft",
+                room_type: "Entire home/apt",
+                bed_type: "Real Bed",
+                address: { street: "Porto, Porto, Portugal" },
+            })
+            .map((x) => x.view(["name"]).name),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+            "Charming & Sunny Mezzanine Loft",
+            "Be Happy in Porto",
+            "Quarry Apartment",
+            "Loft with balcony and view of Casa da Música PORTO",
+            "For 4 or 6 - Wonderful Apartment downtown",
+            "Clérigos Apartments - Apt. 4E for 2/3 Pax",
+            "Bright Top Floor Two Rooms Apartment w/ Terrace",
+            "ReUrban Flats - Santa Catarina #3",
+            "The Blue and White Porto House - Vintage",
+            "4twoLoft at Great Location",
+            "Step In Porto -  Loft w/Mezzanine",
+            "Your OWM Invicta",
+            "Pury Apartments",
+            "Live Porto & Douro - Breiner III",
+        ],
+    );
+    console.timeEnd("query1");
+}
+function query2(qyDB) {
+    console.time("query2");
+    assert.deepEqual(
+        qyDB
+            .queryFiles(
+                "airbnb1",
+                {
+                    property_type: "Loft",
+                    room_type: "Entire home/apt",
+                    bed_type: "Real Bed",
+                    address: { street: "Porto, Porto, Portugal" },
+                },
+                9,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+            "Charming & Sunny Mezzanine Loft",
+            "Be Happy in Porto",
+            "Quarry Apartment",
+            "Loft with balcony and view of Casa da Música PORTO",
+            "For 4 or 6 - Wonderful Apartment downtown",
+            "Clérigos Apartments - Apt. 4E for 2/3 Pax",
+        ],
+    );
+    console.timeEnd("query2");
+}
+function query3(qyDB) {
+    console.time("query3");
+    assert.deepEqual(
+        qyDB
+            .queryFiles(
+                "airbnb1",
+                {
+                    property_type: "Loft",
+                    room_type: "Entire home/apt",
+                    bed_type: "Real Bed",
+                    address: { street: "Porto, Porto, Portugal" },
+                },
+                9,
+                9,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "Bright Top Floor Two Rooms Apartment w/ Terrace",
+            "ReUrban Flats - Santa Catarina #3",
+            "The Blue and White Porto House - Vintage",
+            "4twoLoft at Great Location",
+            "Step In Porto -  Loft w/Mezzanine",
+            "Your OWM Invicta",
+            "Pury Apartments",
+            "Live Porto & Douro - Breiner III",
+        ],
+    );
+    console.timeEnd("query3");
+}
+function query4(qyDB) {
+    console.time("query4");
+    assert.deepEqual(
+        qyDB
+            .queryFiles("airbnb1", {
+                $or: [
+                    {
+                        property_type: "Loft",
+                        room_type: "Entire home/apt",
+                        bed_type: "Real Bed",
+                        address: { street: "Porto, Porto, Portugal" },
+                    },
+                    { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
+                ],
+            })
+            .map((x) => x.view(["name"]).name),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+            "Charming & Sunny Mezzanine Loft",
+            "Brand New Luxury Downtown Studio",
+            "Be Happy in Porto",
+            "Quarry Apartment",
+            "Loft with balcony and view of Casa da Música PORTO",
+            "For 4 or 6 - Wonderful Apartment downtown",
+            "Clérigos Apartments - Apt. 4E for 2/3 Pax",
+            "Bright Top Floor Two Rooms Apartment w/ Terrace",
+            "ReUrban Flats - Santa Catarina #3",
+            "The Blue and White Porto House - Vintage",
+            "4twoLoft at Great Location",
+            "Step In Porto -  Loft w/Mezzanine",
+            "Your OWM Invicta",
+            "Pury Apartments",
+            "Live Porto & Douro - Breiner III",
+        ],
+    );
+    console.timeEnd("query4");
+}
+function query5(qyDB) {
+    console.time("query5");
+    assert.deepEqual(
+        qyDB
+            .queryFiles(
+                "airbnb1",
+                {
+                    $or: [
+                        {
+                            property_type: "Loft",
+                            room_type: "Entire home/apt",
+                            bed_type: "Real Bed",
+                            address: { street: "Porto, Porto, Portugal" },
+                        },
+                        { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
+                    ],
+                },
+                3,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+        ],
+    );
+    console.timeEnd("query5");
+}
+function query6(qyDB) {
+    console.time("query6");
+    assert.deepEqual(
+        qyDB
+            .queryFiles(
+                "airbnb1",
+                {
+                    $or: [
+                        {
+                            property_type: "Loft",
+                            room_type: "Entire home/apt",
+                            bed_type: "Real Bed",
+                            address: { street: "Porto, Porto, Portugal" },
+                        },
+                        { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
+                    ],
+                },
+                20,
+                10,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "Bright Top Floor Two Rooms Apartment w/ Terrace",
+            "ReUrban Flats - Santa Catarina #3",
+            "The Blue and White Porto House - Vintage",
+            "4twoLoft at Great Location",
+            "Step In Porto -  Loft w/Mezzanine",
+            "Your OWM Invicta",
+            "Pury Apartments",
+            "Live Porto & Douro - Breiner III",
+        ],
+    );
+    console.timeEnd("query6");
+}
+function query7(qyDB) {
+    console.time("query7");
+    assert.deepEqual(
+        qyDB
+            .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 })
+            .map((x) => x.view(["name"]).name),
+        ["Romantic loft near historic center", "Brand New Luxury Downtown Studio"],
+    );
+    console.timeEnd("query7");
+}
+function query8(qyDB) {
+    console.time("query8");
+    assert.deepEqual(
+        qyDB
+            .queryFiles("airbnb1", { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 }, 10, 10)
+            .map((x) => x.view(["name"]).name),
+        [],
+    );
+    console.timeEnd("query8");
+}
+function query9(qyDB) {
+    console.time("query9");
+    assert.deepEqual(
+        qyDB
+            .queryFiles(
+                "airbnb1",
+                {
+                    $or: [
+                        {
+                            property_type: "Loft",
+                            room_type: "Entire home/apt",
+                            bed_type: "Real Bed",
+                            address: { street: "Porto, Porto, Portugal" },
+                        },
+                        { property_type: (x) => x == "Loft", minimum_nights: (x) => x >= 60 },
+                    ],
+                },
+                3,
+            )
+            .map((x) => x.view(["name"]).name),
+        [
+            "Porto city centre apartment H",
+            "Romantic loft near historic center",
+            "Lovely sun light  in a stylish loft, Downtown Spot",
+        ],
+    );
+    console.timeEnd("query9");
+}
+function runQueries(qyDB) {
+    query1(qyDB);
+    query1(qyDB);
+    query2(qyDB);
+    query2(qyDB);
+    query3(qyDB);
+    query3(qyDB);
+    query4(qyDB);
+    query4(qyDB);
+    query5(qyDB);
+    query5(qyDB);
+    query6(qyDB);
+    query6(qyDB);
+    query7(qyDB);
+    query7(qyDB);
+    query8(qyDB);
+    query8(qyDB);
+    query9(qyDB);
+    query9(qyDB);
+}
+
+Object.assign(module.exports, { runQueries });

package/scripts/downloadFile.js

@@ -0,0 +1,42 @@
+const fs = require("fs");
+const https = require("https");
+const { ensureParentDir, folderOrFileExists } = require("../src/QyUtils.js");
+const { logger } = require("../src/QyLogger.js");
+
+async function downloadFile(fileUrl, filePath) {
+  if (await folderOrFileExists(filePath)) return;
+  await ensureParentDir(filePath);
+  let resolve, reject;
+  const promise = new Promise((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+  https
+    .get(fileUrl, (response) => {
+      if (response.statusCode !== 200) {
+        logger.error(`Download failed: Status Code ${response.statusCode}`);
+        reject(response.statusCode);
+      } else {
+        const fileStream = fs.createWriteStream(filePath);
+        let accu = 0;
+        response.on("data", (chunk) => {
+          fileStream.write(chunk);
+          accu += chunk.length;
+          if (accu >= 1000_000) {
+            process.stdout.write(".");
+            accu = 0;
+          }
+        });
+        response.on("end", () => {
+          fileStream.close();
+          resolve();
+        });
+      }
+    })
+    .on("error", (err) => {
+      logger.error(`Request error: ${err.message}`);
+      reject(err.message);
+    });
+  return promise;
+}
+Object.assign(module.exports, { downloadFile });

package/scripts/express.js

@@ -0,0 +1,46 @@
+const path = require("node:path");
+const { QyDB } = require("./QyDB.js");
+
+const config = {
+  port: 3629,
+  qyDBDataFolder: "./tmp/QyExpress",
+  staticRoot: path.join(__dirname, "../static/"),
+  rpcFolder: path.join(__dirname, "rpc"),
+};
+
+const { port, qyDBDataFolder } = config;
+const app = require("express")();
+const qyDB = new QyDB(qyDBDataFolder);
+
+let server;
+process.on("SIGINT", async () => {
+  logger.log("SIGINT signal received: closing HTTP server");
+  if (server) {
+    server.close(() => logger.log("HTTP server closed"));
+    server.closeAllConnections();
+  }
+  await qyDB.stop();
+});
+
+run();
+
+async function run() {
+  try {
+    await qyDB.start();
+    server = app.listen(port, () => {
+      logger.log(`Example app listening on port ${port}`);
+    });
+    app.get("/", (request, reply) => {
+      reply.send({ hello: "world from express" });
+    });
+    app.get("/test", (request, reply) => {
+      qyDB.insert("users/test", { a: 3, b: 4 });
+      const content = qyDB.view("users/test");
+      reply.send(content);
+      logger.log(content);
+    });
+  } catch (exception) {
+    logger.error(exception);
+    await qyDB.stop();
+  }
+}

package/scripts/fastify.js

@@ -0,0 +1,79 @@
+"use strict";
+const path = require("node:path");
+const fsPromises = require("node:fs/promises");
+const fastifyStatic = require("@fastify/static");
+const fastifyCompress = require("@fastify/compress");
+const { QyDB } = require("../src/QyDB.js");
+const { folderOrFileExists } = require("../src/QyUtils.js");
+
+const config = {
+  port: 3628,
+  staticRoot: path.join(__dirname, "../static/"),
+  rpcFolder: path.join(__dirname, "../src/rpc"),
+};
+const { port, staticRoot } = config;
+
+const fastify = require("fastify")({ logger: false, forceCloseConnections: true });
+
+const qyDB = new QyDB();
+
+let sigintReceived = false;
+
+process.on("SIGINT", async () => {
+  logger.log("SIGINT signal received: closing HTTP server");
+  if (!sigintReceived) {
+    sigintReceived = true;
+    await fastify.close();
+    await qyDB.stop();
+    logger.log("All closed");
+  }
+});
+
+run();
+
+async function insertRpcs() {
+  const { rpcFolder } = config;
+  if (await folderOrFileExists(rpcFolder)) {
+    await Promise.all(
+      (await fsPromises.readdir(rpcFolder)).map(async (x) => {
+        qyDB.insertRpc(path.basename(x, ".js"), (await fsPromises.readFile(path.join(rpcFolder, x))).toString());
+      }),
+    );
+  }
+}
+
+async function handleGet(request, reply) {
+  const { query, params } = request;
+  const result = await qyDB.callRpc(params.funName, JSON.parse(query.args));
+  if (Error.isError(result)) {
+    logger.warn(result.message);
+  }
+  return reply.send(result);
+}
+
+async function handlePost(request, reply) {
+  logger.log("handling post", request.body);
+  reply.send({ body: request.body });
+}
+
+async function run() {
+  try {
+    await qyDB.start();
+    await insertRpcs();
+    await fastify.register(fastifyStatic, { root: staticRoot });
+    await fastify.register(fastifyCompress);
+    fastify.get("/rpc/:funName", handleGet);
+    fastify.post("/rpc/:funName", handlePost);
+    fastify.listen({ port }, (err) => {
+      if (err) {
+        logger.error(err);
+        process.exit(1);
+      } else {
+        logger.log(`Example app listening on port ${port}`);
+      }
+    });
+  } catch (exception) {
+    logger.error(exception);
+    await qyDB.stop();
+  }
+}

package/scripts/flights.js

@@ -0,0 +1,75 @@
+// node --prof profile.js
+// node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
+const path = require("node:path");
+const fs = require("node:fs");
+const { rm } = require("node:fs/promises");
+const { QyDB } = require("../src/QyDB.js");
+const { folderOrFileExists } = require("../src/QyUtils.js");
+const { runQueries } = require("./flights_queries.js");
+
+logger.level = "info";
+
+const arangoJsonFolderPath = "../GraphCourse_DemoData_ArangoDB-2";
+const dataFolder = "./tmp/flights";
+
+const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
+
+run();
+
+function insertAirports() {
+    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "airports.json")));
+    console.time("insertAirports");
+    const { batch } = qyDB;
+    for (const record of data) {
+        batch.insert(`airports/${record._key}`, record);
+    }
+    batch.run();
+    console.timeEnd("insertAirports");
+}
+
+function insertFlights() {
+    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "flights.json")));
+    console.time("insertFlights");
+    const { batch } = qyDB;
+    for (let i = 0; i < data.length; ++i) {
+        const record = data[i];
+        // const filePath = `flights/${record._from}/${record._to}/${i}`;
+        const filePath = `flights/${i}`;
+        batch.insert(filePath, record);
+    }
+    batch.run();
+    console.timeEnd("insertFlights");
+    console.time("createIndex");
+    // qyDB.createIndex("flights", ["_from", "_to"]);
+    qyDB.createIndex("flights", ["_from", "_to", "Month", "Day", "Year"]);
+    console.timeEnd("createIndex");
+}
+
+async function run() {
+    console.time("total");
+    if (process.argv[3]) {
+        logger.level = process.argv[3];
+    }
+    let shouldInsertData;
+    if (process.argv[2] == "true") {
+        try {
+            await rm(dataFolder, { recursive: true });
+        } catch (exception) {}
+        shouldInsertData = true;
+    } else if (!(await folderOrFileExists(dataFolder))) shouldInsertData = true;
+    console.time("run");
+    console.time("start");
+    await qyDB.start();
+    console.timeEnd("start");
+    if (shouldInsertData) {
+        insertAirports();
+        insertFlights();
+    }
+    // console.log(qyDB.dir("/%/index/0dJIntKA55/_from").subdirNameList);
+    runQueries(qyDB);
+    console.time("stop");
+    await qyDB.stop();
+    console.timeEnd("stop");
+    console.timeEnd("run");
+    console.timeEnd("total");
+}

package/scripts/flights2.js

@@ -0,0 +1,81 @@
+// node --prof profile.js
+// node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
+const path = require("node:path");
+const fs = require("node:fs");
+const { rm } = require("node:fs/promises");
+const { QyDB } = require("../src/QyDB.js");
+const { folderOrFileExists } = require("../src/QyUtils.js");
+const { runQueries } = require("./flights2_queries.js");
+
+logger.level = "info";
+
+const arangoJsonFolderPath = "../GraphCourse_DemoData_ArangoDB-2";
+const dataFolder = "./tmp/flights2";
+const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
+
+run();
+
+function insertAirports() {
+    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "airports.json")));
+    console.time("insertAirports");
+    const { batch } = qyDB;
+    for (const record of data) {
+        const { country, state, city } = record;
+        batch.insert(`airports/${country}/${state}/${city}/${record._key}`, record);
+    }
+    batch.run();
+    console.timeEnd("insertAirports");
+}
+
+function insertFlights() {
+    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "flights.json")));
+    console.time("insertFlights");
+    const { batch } = qyDB;
+    for (let i = 0; i < data.length; ++i) {
+        const record = data[i];
+        const { Year, Month, Day } = record;
+        batch.insert(`flights/${Year}/${Month}/${Day}/${i}`, record);
+        // batch.insertP(["flights", Year, Month, Day, i], record);
+    }
+    batch.run();
+    console.timeEnd("insertFlights");
+    console.time("createFlightsIndex");
+    // qyDB.createIndex("flights", ["_from", "_to"]);
+    for (const yearDir of qyDB.dir("flights").subdirList) {
+        for (const monthDir of yearDir.subdirList) {
+            for (const dayDir of monthDir.subdirList) {
+                batch.createIndex(dayDir, ["_from", "_to"]);
+            }
+        }
+    }
+    batch.run();
+    console.timeEnd("createFlightsIndex");
+}
+
+async function run() {
+    console.time("total");
+    if (process.argv[3]) {
+        logger.level = process.argv[3];
+    }
+    let shouldInsertData;
+    if (process.argv[2] == "true") {
+        try {
+            await rm(dataFolder, { recursive: true });
+        } catch (exception) {}
+        shouldInsertData = true;
+    } else if (!(await folderOrFileExists(dataFolder))) shouldInsertData = true;
+    console.time("run");
+    console.time("start");
+    await qyDB.start();
+    console.timeEnd("start");
+    if (shouldInsertData) {
+        insertAirports();
+        insertFlights();
+    }
+    runQueries(qyDB);
+    console.time("stop");
+    await qyDB.stop();
+    console.timeEnd("stop");
+    console.timeEnd("run");
+    console.timeEnd("total");
+}