datafolder 1.0.7 → 1.0.9

package/scripts/downloadFile.js

@@ -1,40 +1,41 @@
-import { createWriteStream } from "fs";
-import { get } from "https";
-import { ensureParentDir, folderOrFileExists } from "../src/QyUtils.js";
-import { logger } from "../src/QyLogger.js";
-
-async function downloadFile(fileUrl, filePath) {
-  if (await folderOrFileExists(filePath)) return;
-  await ensureParentDir(filePath);
-  let resolve, reject;
-  const promise = new Promise((res, rej) => {
-    resolve = res;
-    reject = rej;
-  });
-  get(fileUrl, (response) => {
-    if (response.statusCode !== 200) {
-      logger.error(`Download failed: Status Code ${response.statusCode}`);
-      reject(response.statusCode);
-    } else {
-      const fileStream = createWriteStream(filePath);
-      let accu = 0;
-      response.on("data", (chunk) => {
-        fileStream.write(chunk);
-        accu += chunk.length;
-        if (accu >= 1000_000) {
-          process.stdout.write(".");
-          accu = 0;
-        }
-      });
-      response.on("end", () => {
-        fileStream.close();
-        resolve();
-      });
-    }
-  }).on("error", (err) => {
-    logger.error(`Request error: ${err.message}`);
-    reject(err.message);
-  });
-  return promise;
-}
-export { downloadFile };
+import { createWriteStream } from "fs";
+import { get } from "https";
+import { ensureParentDir, folderOrFileExists } from "../src/QyUtils.js";
+import { logger } from "../src/QyLogger.js";
+async function downloadFile(fileUrl, filePath) {
+    if (await folderOrFileExists(filePath))
+        return;
+    await ensureParentDir(filePath);
+    let resolve, reject;
+    const promise = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    get(fileUrl, (response) => {
+        if (response.statusCode !== 200) {
+            logger.error(`Download failed: Status Code ${response.statusCode}`);
+            reject(response.statusCode);
+        }
+        else {
+            const fileStream = createWriteStream(filePath);
+            let accu = 0;
+            response.on("data", (chunk) => {
+                fileStream.write(chunk);
+                accu += chunk.length;
+                if (accu >= 1000_000) {
+                    process.stdout.write(".");
+                    accu = 0;
+                }
+            });
+            response.on("end", () => {
+                fileStream.close();
+                resolve();
+            });
+        }
+    }).on("error", (err) => {
+        logger.error(`Request error: ${err.message}`);
+        reject(err.message);
+    });
+    return promise;
+}
+export { downloadFile };

package/scripts/express.js

@@ -1,48 +1,44 @@
-import { join } from "node:path";
-import { QyDB } from "../src/QyDB.js";
-import Express from "express";
-import { __dirname } from "../src/QyUtils.js";
-
-const config = {
-  port: 3629,
-  qyDBDataFolder: "./tmp/QyExpress",
-  staticRoot: join(__dirname, "../static/"),
-  rpcFolder: join(__dirname, "rpc"),
-};
-
-const { port, qyDBDataFolder } = config;
-const app = Express();
-const qyDB = new QyDB(qyDBDataFolder);
-
-let server;
-process.on("SIGINT", async () => {
-  logger.log("SIGINT signal received: closing HTTP server");
-  if (server) {
-    server.close(() => logger.log("HTTP server closed"));
-    server.closeAllConnections();
-  }
-  await qyDB.stop();
-});
-
-run();
-
-async function run() {
-  try {
-    await qyDB.start();
-    server = app.listen(port, () => {
-      logger.log(`Example app listening on port ${port}`);
-    });
-    app.get("/", (request, reply) => {
-      reply.send({ hello: "world from express" });
-    });
-    app.get("/test", (request, reply) => {
-      qyDB.insert("users/test", { a: 3, b: 4 });
-      const content = qyDB.view("users/test");
-      reply.send(content);
-      logger.log(content);
-    });
-  } catch (exception) {
-    logger.error(exception);
-    await qyDB.stop();
-  }
-}
+import { join } from "node:path";
+import { QyDB } from "../src/QyDB.js";
+import Express from "express";
+import { __dirname } from "../src/QyUtils.js";
+const config = {
+    port: 3629,
+    qyDBDataFolder: "./tmp/QyExpress",
+    staticRoot: join(__dirname, "../static/"),
+    rpcFolder: join(__dirname, "rpc"),
+};
+const { port, qyDBDataFolder } = config;
+const app = Express();
+const qyDB = new QyDB(qyDBDataFolder);
+let server;
+process.on("SIGINT", async () => {
+    logger.log("SIGINT signal received: closing HTTP server");
+    if (server) {
+        server.close(() => logger.log("HTTP server closed"));
+        server.closeAllConnections();
+    }
+    await qyDB.stop();
+});
+run();
+async function run() {
+    try {
+        await qyDB.start();
+        server = app.listen(port, () => {
+            logger.log(`Example app listening on port ${port}`);
+        });
+        app.get("/", (request, reply) => {
+            reply.send({ hello: "world from express" });
+        });
+        app.get("/test", (request, reply) => {
+            qyDB.insert("users/test", { a: 3, b: 4 });
+            const content = qyDB.view("users/test");
+            reply.send(content);
+            logger.log(content);
+        });
+    }
+    catch (exception) {
+        logger.error(exception);
+        await qyDB.stop();
+    }
+}

package/scripts/fastify.js

@@ -1,80 +1,71 @@
-"use strict";
-import { join, basename } from "node:path";
-import { readdir, readFile } from "node:fs/promises";
-import fastifyStatic from "@fastify/static";
-import fastifyCompress from "@fastify/compress";
-import { QyDB } from "../src/QyDB.js";
-import { folderOrFileExists, __dirname } from "../src/QyUtils.js";
-import Fastify from "fastify";
-
-const config = {
-  port: 3628,
-  staticRoot: join(__dirname, "../static/"),
-  rpcFolder: join(__dirname, "../src/rpc"),
-};
-const { port, staticRoot } = config;
-
-const fastify = Fastify({ logger: false, forceCloseConnections: true });
-
-const qyDB = new QyDB(join(__dirname, "../tmp/QyTrigger/1"));
-
-let sigintReceived = false;
-
-process.on("SIGINT", async () => {
-  logger.log("SIGINT signal received: closing HTTP server");
-  if (!sigintReceived) {
-    sigintReceived = true;
-    await fastify.close();
-    await qyDB.stop();
-    logger.log("All closed");
-  }
-});
-
-run();
-
-async function insertRpcs() {
-  const { rpcFolder } = config;
-  if (await folderOrFileExists(rpcFolder)) {
-    await Promise.all(
-      (await readdir(rpcFolder)).map(async (x) => {
-        qyDB.insertRpc(basename(x, ".js"), (await readFile(join(rpcFolder, x))).toString());
-      }),
-    );
-  }
-}
-
-async function handleGet(request, reply) {
-  const { query, params } = request;
-  const result = await qyDB.callRpc(params.funName, query && query.args ? JSON.parse(query.args) : undefined);
-  if (Error.isError(result)) {
-    logger.warn(result.message);
-  }
-  return reply.send(result);
-}
-
-async function handlePost(request, reply) {
-  logger.log("handling post", request.body);
-  reply.send({ body: request.body });
-}
-
-async function run() {
-  try {
-    await qyDB.start();
-    await insertRpcs();
-    await fastify.register(fastifyStatic, { root: staticRoot });
-    await fastify.register(fastifyCompress);
-    fastify.get("/rpc/:funName", handleGet);
-    fastify.post("/rpc/:funName", handlePost);
-    fastify.listen({ port }, (err) => {
-      if (err) {
-        logger.error(err);
-        process.exit(1);
-      } else {
-        logger.log(`Example app listening on port ${port}`);
-      }
-    });
-  } catch (exception) {
-    logger.error(exception);
-    await qyDB.stop();
-  }
-}
+"use strict";
+import { join, basename } from "node:path";
+import { readdir, readFile } from "node:fs/promises";
+import fastifyStatic from "@fastify/static";
+import fastifyCompress from "@fastify/compress";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists, __dirname } from "../src/QyUtils.js";
+import Fastify from "fastify";
+const config = {
+    port: 3628,
+    staticRoot: join(__dirname, "../static/"),
+    rpcFolder: join(__dirname, "../src/rpc"),
+};
+const { port, staticRoot } = config;
+const fastify = Fastify({ logger: false, forceCloseConnections: true });
+// const qyDB = new QyDB(join(__dirname, "../tmp/simulate2"));
+const qyDB = new QyDB();
+let sigintReceived = false;
+process.on("SIGINT", async () => {
+    logger.log("SIGINT signal received: closing HTTP server");
+    if (!sigintReceived) {
+        sigintReceived = true;
+        await fastify.close();
+        await qyDB.stop();
+        logger.log("All closed");
+    }
+});
+run();
+async function insertRpcs() {
+    const { rpcFolder } = config;
+    if (await folderOrFileExists(rpcFolder)) {
+        await Promise.all((await readdir(rpcFolder)).map(async (x) => {
+            qyDB.insertRpc(basename(x, ".js"), (await readFile(join(rpcFolder, x))).toString());
+        }));
+    }
+}
+async function handleGet(request, reply) {
+    const { query, params } = request;
+    const result = await qyDB.callRpc(params.funName, query && query.args ? JSON.parse(query.args) : undefined);
+    if (Error.isError(result)) {
+        logger.warn(result.message);
+    }
+    return reply.send(result);
+}
+async function handlePost(request, reply) {
+    logger.log("handling post", request.body);
+    reply.send({ body: request.body });
+}
+async function run() {
+    try {
+        await qyDB.start();
+        await insertRpcs();
+        await fastify.register(fastifyStatic, { root: staticRoot });
+        await fastify.register(fastifyCompress);
+        fastify.get("/rpc/:funName", handleGet);
+        fastify.post("/rpc/:funName", handlePost);
+        fastify.listen({ port }, (err) => {
+            if (err) {
+                logger.error(err);
+                process.exit(1);
+            }
+            else {
+                logger.log(`Example app listening on port ${port}`);
+            }
+        });
+    }
+    catch (exception) {
+        logger.error(exception);
+        await qyDB.stop();
+    }
+}

package/scripts/flights.js

@@ -1,75 +1,71 @@
-// node --prof profile.js
-// node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
-import { join } from "node:path";
-import { readFileSync } from "node:fs";
-import { rm } from "node:fs/promises";
-import { QyDB } from "../src/QyDB.js";
-import { folderOrFileExists } from "../src/QyUtils.js";
-import { runQueries } from "./flights_queries.js";
-
-logger.level = "info";
-
-const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
-const dataFolder = "./tmp/flights";
-
-const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
-
-run();
-
-function insertAirports() {
-    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
-    console.time("insertAirports");
-    const { batch } = qyDB;
-    for (const record of data) {
-        batch.insert(`airports/${record._key}`, record);
-    }
-    batch.run();
-    console.timeEnd("insertAirports");
-}
-
-function insertFlights() {
-    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "flights.json")));
-    console.time("insertFlights");
-    const { batch } = qyDB;
-    for (let i = 0; i < data.length; ++i) {
-        const record = data[i];
-        // const filePath = `flights/${record._from}/${record._to}/${i}`;
-        const filePath = `flights/${i}`;
-        batch.insert(filePath, record);
-    }
-    batch.run();
-    console.timeEnd("insertFlights");
-    console.time("createIndex");
-    // qyDB.createIndex("flights", ["_from", "_to"]);
-    qyDB.createIndex("flights", ["_from", "_to", "Month", "Day", "Year"]);
-    console.timeEnd("createIndex");
-}
-
-async function run() {
-    console.time("total");
-    if (process.argv[3]) {
-        logger.level = process.argv[3];
-    }
-    let shouldInsertData;
-    if (process.argv[2] == "true") {
-        try {
-            await rm(dataFolder, { recursive: true });
-        } catch (exception) {}
-        shouldInsertData = true;
-    } else if (!(await folderOrFileExists(dataFolder))) shouldInsertData = true;
-    console.time("run");
-    console.time("start");
-    await qyDB.start();
-    console.timeEnd("start");
-    if (shouldInsertData) {
-        insertAirports();
-        insertFlights();
-    }
-    // console.log(qyDB.dir("/%/index/0dJIntKA55/_from").subdirNameList);
-    runQueries(qyDB);
-    console.time("stop");
-    await qyDB.stop();
-    console.timeEnd("stop");
-    console.timeEnd("run");
-    console.timeEnd("total");
-}
+// node --prof profile.js
+// node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
+import { join } from "node:path";
+import { readFileSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists } from "../src/QyUtils.js";
+import { runQueries } from "./flights_queries.js";
+logger.level = "info";
+const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
+const dataFolder = "./tmp/flights";
+const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
+run();
+function insertAirports() {
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
+    console.time("insertAirports");
+    const { batch } = qyDB;
+    for (const record of data) {
+        batch.insert(`airports/${record._key}`, record);
+    }
+    batch.run();
+    console.timeEnd("insertAirports");
+}
+function insertFlights() {
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "flights.json")));
+    console.time("insertFlights");
+    const { batch } = qyDB;
+    for (let i = 0; i < data.length; ++i) {
+        const record = data[i];
+        // const filePath = `flights/${record._from}/${record._to}/${i}`;
+        const filePath = `flights/${i}`;
+        batch.insert(filePath, record);
+    }
+    batch.run();
+    console.timeEnd("insertFlights");
+    console.time("createIndex");
+    // qyDB.createIndex("flights", ["_from", "_to"]);
+    qyDB.createIndex("flights", ["_from", "_to", "Month", "Day", "Year"]);
+    console.timeEnd("createIndex");
+}
+async function run() {
+    console.time("total");
+    if (process.argv[3]) {
+        logger.level = process.argv[3];
+    }
+    let shouldInsertData;
+    if (process.argv[2] == "true") {
+        try {
+            await rm(dataFolder, { recursive: true });
+        }
+        catch (exception) { }
+        shouldInsertData = true;
+    }
+    else if (!(await folderOrFileExists(dataFolder)))
+        shouldInsertData = true;
+    console.time("run");
+    console.time("start");
+    await qyDB.start();
+    console.timeEnd("start");
+    if (shouldInsertData) {
+        insertAirports();
+        insertFlights();
+    }
+    // console.log(qyDB.dir("/%/index/0dJIntKA55/_from").subdirNameList);
+    runQueries(qyDB);
+    console.time("stop");
+    await qyDB.stop();
+    console.timeEnd("stop");
+    console.timeEnd("run");
+    console.timeEnd("total");
+}

package/scripts/flights2.js

@@ -1,81 +1,78 @@
-// node --prof profile.js
-// node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
-import { join } from "node:path";
-import { readFileSync } from "node:fs";
-import { rm } from "node:fs/promises";
-import { QyDB } from "../src/QyDB.js";
-import { folderOrFileExists } from "../src/QyUtils.js";
-import { runQueries } from "./flights2_queries.js";
-
-logger.level = "info";
-
-const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
-const dataFolder = "./tmp/flights2";
-const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
-
-run();
-
-function insertAirports() {
-    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
-    console.time("insertAirports");
-    const { batch } = qyDB;
-    for (const record of data) {
-        const { country, state, city } = record;
-        batch.insert(`airports/${country}/${state}/${city}/${record._key}`, record);
-    }
-    batch.run();
-    console.timeEnd("insertAirports");
-}
-
-function insertFlights() {
-    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "flights.json")));
-    console.time("insertFlights");
-    const { batch } = qyDB;
-    for (let i = 0; i < data.length; ++i) {
-        const record = data[i];
-        const { Year, Month, Day } = record;
-        batch.insert(`flights/${Year}/${Month}/${Day}/${i}`, record);
-        // batch.insertP(["flights", Year, Month, Day, i], record);
-    }
-    batch.run();
-    console.timeEnd("insertFlights");
-    console.time("createFlightsIndex");
-    // qyDB.createIndex("flights", ["_from", "_to"]);
-    for (const yearDir of qyDB.dir("flights").subdirList) {
-        for (const monthDir of yearDir.subdirList) {
-            for (const dayDir of monthDir.subdirList) {
-                batch.createIndex(dayDir, ["_from", "_to"]);
-            }
-        }
-    }
-    batch.run();
-    console.timeEnd("createFlightsIndex");
-}
-
-async function run() {
-    console.time("total");
-    if (process.argv[3]) {
-        logger.level = process.argv[3];
-    }
-    let shouldInsertData;
-    if (process.argv[2] == "true") {
-        try {
-            await rm(dataFolder, { recursive: true });
-        } catch (exception) {}
-        shouldInsertData = true;
-    } else if (!(await folderOrFileExists(dataFolder))) shouldInsertData = true;
-    console.time("run");
-    console.time("start");
-    await qyDB.start();
-    console.timeEnd("start");
-    if (shouldInsertData) {
-        insertAirports();
-        insertFlights();
-    }
-    runQueries(qyDB);
-    console.time("stop");
-    await qyDB.stop();
-    console.timeEnd("stop");
-    console.timeEnd("run");
-    console.timeEnd("total");
-}
+// node --prof profile.js
+// node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
+import { join } from "node:path";
+import { readFileSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists } from "../src/QyUtils.js";
+import { runQueries } from "./flights2_queries.js";
+logger.level = "info";
+const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
+const dataFolder = "./tmp/flights2";
+const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
+run();
+function insertAirports() {
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
+    console.time("insertAirports");
+    const { batch } = qyDB;
+    for (const record of data) {
+        const { country, state, city } = record;
+        batch.insert(`airports/${country}/${state}/${city}/${record._key}`, record);
+    }
+    batch.run();
+    console.timeEnd("insertAirports");
+}
+function insertFlights() {
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "flights.json")));
+    console.time("insertFlights");
+    const { batch } = qyDB;
+    for (let i = 0; i < data.length; ++i) {
+        const record = data[i];
+        const { Year, Month, Day } = record;
+        batch.insert(`flights/${Year}/${Month}/${Day}/${i}`, record);
+        // batch.insertP(["flights", Year, Month, Day, i], record);
+    }
+    batch.run();
+    console.timeEnd("insertFlights");
+    console.time("createFlightsIndex");
+    // qyDB.createIndex("flights", ["_from", "_to"]);
+    for (const yearDir of qyDB.dir("flights").subdirList) {
+        for (const monthDir of yearDir.subdirList) {
+            for (const dayDir of monthDir.subdirList) {
+                batch.createIndex(dayDir, ["_from", "_to"]);
+            }
+        }
+    }
+    batch.run();
+    console.timeEnd("createFlightsIndex");
+}
+async function run() {
+    console.time("total");
+    if (process.argv[3]) {
+        logger.level = process.argv[3];
+    }
+    let shouldInsertData;
+    if (process.argv[2] == "true") {
+        try {
+            await rm(dataFolder, { recursive: true });
+        }
+        catch (exception) { }
+        shouldInsertData = true;
+    }
+    else if (!(await folderOrFileExists(dataFolder)))
+        shouldInsertData = true;
+    console.time("run");
+    console.time("start");
+    await qyDB.start();
+    console.timeEnd("start");
+    if (shouldInsertData) {
+        insertAirports();
+        insertFlights();
+    }
+    runQueries(qyDB);
+    console.time("stop");
+    await qyDB.stop();
+    console.timeEnd("stop");
+    console.timeEnd("run");
+    console.timeEnd("total");
+}