datafolder 1.0.2 → 1.0.4

package/scripts/downloadFile.js

@@ -1,7 +1,7 @@
-const fs = require("fs");
-const https = require("https");
-const { ensureParentDir, folderOrFileExists } = require("../src/QyUtils.js");
-const { logger } = require("../src/QyLogger.js");
+import { createWriteStream } from "fs";
+import { get } from "https";
+import { ensureParentDir, folderOrFileExists } from "../src/QyUtils.js";
+import { logger } from "../src/QyLogger.js";
 
 async function downloadFile(fileUrl, filePath) {
   if (await folderOrFileExists(filePath)) return;
@@ -11,32 +11,30 @@ async function downloadFile(fileUrl, filePath) {
     resolve = res;
     reject = rej;
   });
-  https
-    .get(fileUrl, (response) => {
-      if (response.statusCode !== 200) {
-        logger.error(`Download failed: Status Code ${response.statusCode}`);
-        reject(response.statusCode);
-      } else {
-        const fileStream = fs.createWriteStream(filePath);
-        let accu = 0;
-        response.on("data", (chunk) => {
-          fileStream.write(chunk);
-          accu += chunk.length;
-          if (accu >= 1000_000) {
-            process.stdout.write(".");
-            accu = 0;
-          }
-        });
-        response.on("end", () => {
-          fileStream.close();
-          resolve();
-        });
-      }
-    })
-    .on("error", (err) => {
-      logger.error(`Request error: ${err.message}`);
-      reject(err.message);
-    });
+  get(fileUrl, (response) => {
+    if (response.statusCode !== 200) {
+      logger.error(`Download failed: Status Code ${response.statusCode}`);
+      reject(response.statusCode);
+    } else {
+      const fileStream = createWriteStream(filePath);
+      let accu = 0;
+      response.on("data", (chunk) => {
+        fileStream.write(chunk);
+        accu += chunk.length;
+        if (accu >= 1000_000) {
+          process.stdout.write(".");
+          accu = 0;
+        }
+      });
+      response.on("end", () => {
+        fileStream.close();
+        resolve();
+      });
+    }
+  }).on("error", (err) => {
+    logger.error(`Request error: ${err.message}`);
+    reject(err.message);
+  });
   return promise;
 }
-Object.assign(module.exports, { downloadFile });
+export { downloadFile };

package/scripts/express.js

@@ -1,15 +1,17 @@
-const path = require("node:path");
-const { QyDB } = require("./QyDB.js");
+import { join } from "node:path";
+import { QyDB } from "../src/QyDB.js";
+import Express from "express";
+import { __dirname } from "../src/QyUtils.js";
 
 const config = {
   port: 3629,
   qyDBDataFolder: "./tmp/QyExpress",
-  staticRoot: path.join(__dirname, "../static/"),
-  rpcFolder: path.join(__dirname, "rpc"),
+  staticRoot: join(__dirname, "../static/"),
+  rpcFolder: join(__dirname, "rpc"),
 };
 
 const { port, qyDBDataFolder } = config;
-const app = require("express")();
+const app = Express();
 const qyDB = new QyDB(qyDBDataFolder);
 
 let server;

package/scripts/fastify.js

@@ -1,19 +1,20 @@
 "use strict";
-const path = require("node:path");
-const fsPromises = require("node:fs/promises");
-const fastifyStatic = require("@fastify/static");
-const fastifyCompress = require("@fastify/compress");
-const { QyDB } = require("../src/QyDB.js");
-const { folderOrFileExists } = require("../src/QyUtils.js");
+import { join, basename } from "node:path";
+import { readdir, readFile } from "node:fs/promises";
+import fastifyStatic from "@fastify/static";
+import fastifyCompress from "@fastify/compress";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists, __dirname } from "../src/QyUtils.js";
+import Fastify from "fastify";
 
 const config = {
   port: 3628,
-  staticRoot: path.join(__dirname, "../static/"),
-  rpcFolder: path.join(__dirname, "../src/rpc"),
+  staticRoot: join(__dirname, "../static/"),
+  rpcFolder: join(__dirname, "../src/rpc"),
 };
 const { port, staticRoot } = config;
 
-const fastify = require("fastify")({ logger: false, forceCloseConnections: true });
+const fastify = Fastify({ logger: false, forceCloseConnections: true });
 
 const qyDB = new QyDB();
 
@@ -35,8 +36,8 @@ async function insertRpcs() {
   const { rpcFolder } = config;
   if (await folderOrFileExists(rpcFolder)) {
     await Promise.all(
-      (await fsPromises.readdir(rpcFolder)).map(async (x) => {
-        qyDB.insertRpc(path.basename(x, ".js"), (await fsPromises.readFile(path.join(rpcFolder, x))).toString());
+      (await readdir(rpcFolder)).map(async (x) => {
+        qyDB.insertRpc(basename(x, ".js"), (await readFile(join(rpcFolder, x))).toString());
       }),
     );
   }

package/scripts/flights.js

@@ -1,15 +1,15 @@
 // node --prof profile.js
 // node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
-const path = require("node:path");
-const fs = require("node:fs");
-const { rm } = require("node:fs/promises");
-const { QyDB } = require("../src/QyDB.js");
-const { folderOrFileExists } = require("../src/QyUtils.js");
-const { runQueries } = require("./flights_queries.js");
+import { join } from "node:path";
+import { readFileSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists } from "../src/QyUtils.js";
+import { runQueries } from "./flights_queries.js";
 
 logger.level = "info";
 
-const arangoJsonFolderPath = "../GraphCourse_DemoData_ArangoDB-2";
+const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
 const dataFolder = "./tmp/flights";
 
 const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
@@ -17,7 +17,7 @@ const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
 run();
 
 function insertAirports() {
-    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "airports.json")));
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
     console.time("insertAirports");
     const { batch } = qyDB;
     for (const record of data) {
@@ -28,7 +28,7 @@ function insertAirports() {
 }
 
 function insertFlights() {
-    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "flights.json")));
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "flights.json")));
     console.time("insertFlights");
     const { batch } = qyDB;
     for (let i = 0; i < data.length; ++i) {

package/scripts/flights2.js

@@ -1,22 +1,22 @@
 // node --prof profile.js
 // node --prof-process isolate-0xnnnnnnnnnnnn-v8.log > processed.txt
-const path = require("node:path");
-const fs = require("node:fs");
-const { rm } = require("node:fs/promises");
-const { QyDB } = require("../src/QyDB.js");
-const { folderOrFileExists } = require("../src/QyUtils.js");
-const { runQueries } = require("./flights2_queries.js");
+import { join } from "node:path";
+import { readFileSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import { QyDB } from "../src/QyDB.js";
+import { folderOrFileExists } from "../src/QyUtils.js";
+import { runQueries } from "./flights2_queries.js";
 
 logger.level = "info";
 
-const arangoJsonFolderPath = "../GraphCourse_DemoData_ArangoDB-2";
+const arangoJsonFolderPath = "D:\\scripts\\GraphCourse_DemoData_ArangoDB-2";
 const dataFolder = "./tmp/flights2";
 const qyDB = new QyDB(dataFolder, { preloadFoldersAtStart: false });
 
 run();
 
 function insertAirports() {
-    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "airports.json")));
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "airports.json")));
     console.time("insertAirports");
     const { batch } = qyDB;
     for (const record of data) {
@@ -28,7 +28,7 @@ function insertAirports() {
 }
 
 function insertFlights() {
-    const data = JSON.parse(fs.readFileSync(path.join(arangoJsonFolderPath, "flights.json")));
+    const data = JSON.parse(readFileSync(join(arangoJsonFolderPath, "flights.json")));
     console.time("insertFlights");
     const { batch } = qyDB;
     for (let i = 0; i < data.length; ++i) {

package/scripts/flights2_queries.js

@@ -1,4 +1,4 @@
-const assert = require("node:assert/strict");
+import { deepEqual } from "node:assert/strict";
 
 // Return the names of all airports one can reach
 // directly (1 step) from Los Angeles International
@@ -6,94 +6,93 @@ const assert = require("node:assert/strict");
 
 function query1(qyDB) {
     console.time("query1");
-    const resultMap = {};
-    for (const x of qyDB.queryFiles("flights/**", { _from: "airports/LAX" })) {
-        resultMap[x.fileContent._to] = 1;
-    }
-    assert.deepEqual(Object.keys(resultMap), [
-        "airports/CVG",
-        "airports/SMF",
-        "airports/ATL",
-        "airports/MSP",
-        "airports/DFW",
-        "airports/PHL",
-        "airports/IAH",
-        "airports/TUS",
-        "airports/ORD",
-        "airports/PDX",
-        "airports/MIA",
-        "airports/SFO",
-        "airports/DEN",
-        "airports/IAD",
-        "airports/SJC",
-        "airports/PHX",
-        "airports/SEA",
-        "airports/SLC",
-        "airports/JFK",
-        "airports/SAN",
-        "airports/ONT",
-        "airports/LAS",
-        "airports/MCI",
-        "airports/MDW",
-        "airports/FLL",
-        "airports/AUS",
-        "airports/EWR",
-        "airports/CLE",
-        "airports/SBP",
-        "airports/MEM",
-        "airports/SBA",
-        "airports/CLT",
-        "airports/COS",
-        "airports/ASE",
-        "airports/ICT",
-        "airports/HOU",
-        "airports/YUM",
-        "airports/STL",
-        "airports/FAT",
-        "airports/ABQ",
-        "airports/BOI",
-        "airports/BOS",
-        "airports/LIH",
-        "airports/HNL",
-        "airports/SMX",
-        "airports/MRY",
-        "airports/DTW",
-        "airports/RNO",
-        "airports/MCO",
-        "airports/OXR",
-        "airports/OAK",
-        "airports/OGG",
-        "airports/KOA",
-        "airports/BWI",
-        "airports/CLD",
-        "airports/IND",
-        "airports/PSP",
-        "airports/ELP",
-        "airports/TPA",
-        "airports/OKC",
-        "airports/MSY",
-        "airports/SNA",
-        "airports/IYK",
-        "airports/IPL",
-        "airports/SAT",
-        "airports/MOD",
-        "airports/BNA",
-        "airports/DCA",
-        "airports/SGU",
-        "airports/BFL",
-        "airports/EUG",
-        "airports/EGE",
-        "airports/GEG",
-        "airports/BDL",
-        "airports/JAX",
-        "airports/CMH",
-        "airports/PIT",
-        "airports/RDU",
-        "airports/SJU",
-        "airports/XNA",
-        "airports/MTJ",
-        "airports/JAC",
-    ]);
+    deepEqual(
+        [...new Set(qyDB.queryFiles("flights/**", { _from: "airports/LAX" }).map((x) => x.prop("_to"))).values()],
+        [
+            "airports/CVG",
+            "airports/SMF",
+            "airports/ATL",
+            "airports/MSP",
+            "airports/DFW",
+            "airports/PHL",
+            "airports/IAH",
+            "airports/TUS",
+            "airports/ORD",
+            "airports/PDX",
+            "airports/MIA",
+            "airports/SFO",
+            "airports/DEN",
+            "airports/IAD",
+            "airports/SJC",
+            "airports/PHX",
+            "airports/SEA",
+            "airports/SLC",
+            "airports/JFK",
+            "airports/SAN",
+            "airports/ONT",
+            "airports/LAS",
+            "airports/MCI",
+            "airports/MDW",
+            "airports/FLL",
+            "airports/AUS",
+            "airports/EWR",
+            "airports/CLE",
+            "airports/SBP",
+            "airports/MEM",
+            "airports/SBA",
+            "airports/CLT",
+            "airports/COS",
+            "airports/ASE",
+            "airports/ICT",
+            "airports/HOU",
+            "airports/YUM",
+            "airports/STL",
+            "airports/FAT",
+            "airports/ABQ",
+            "airports/BOI",
+            "airports/BOS",
+            "airports/LIH",
+            "airports/HNL",
+            "airports/SMX",
+            "airports/MRY",
+            "airports/DTW",
+            "airports/RNO",
+            "airports/MCO",
+            "airports/OXR",
+            "airports/OAK",
+            "airports/OGG",
+            "airports/KOA",
+            "airports/BWI",
+            "airports/CLD",
+            "airports/IND",
+            "airports/PSP",
+            "airports/ELP",
+            "airports/TPA",
+            "airports/OKC",
+            "airports/MSY",
+            "airports/SNA",
+            "airports/IYK",
+            "airports/IPL",
+            "airports/SAT",
+            "airports/MOD",
+            "airports/BNA",
+            "airports/DCA",
+            "airports/SGU",
+            "airports/BFL",
+            "airports/EUG",
+            "airports/EGE",
+            "airports/GEG",
+            "airports/BDL",
+            "airports/JAX",
+            "airports/CMH",
+            "airports/PIT",
+            "airports/RDU",
+            "airports/SJU",
+            "airports/XNA",
+            "airports/MTJ",
+            "airports/JAC",
+        ],
+    );
     console.timeEnd("query1");
 }
 
@@ -111,24 +110,25 @@ function query1(qyDB) {
 
 function query2(qyDB) {
     console.time("query2");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(["flights", "*", 1, (x) => x >= 5 && x <= 7], {
                 $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
             })
-            .map((x) => x.fileContent.FlightNum),
+            .map((x) => x.prop("FlightNum"))
+            .sort((x, y) => x - y),
         [
             6635, 646, 6541, 5651, 6541, 5648, 687, 309, 6678, 6678, 6636, 439, 1567, 6635, 5647, 6541, 5660, 6541,
             6553, 687, 6553, 309, 5649, 5650, 6636, 439, 6635, 574, 6541, 5651, 5648, 6541, 1705, 1535, 5649, 6678,
             5650, 6636, 6678, 1087,
-        ],
+        ].sort((x, y) => x - y),
     );
     console.timeEnd("query2");
 }
 
 function query3(qyDB) {
     console.time("query3");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(["flights", "*", 1, (x) => x >= 1 && x <= 7], {
                 $or: [
@@ -136,7 +136,8 @@ function query3(qyDB) {
                     { _from: "airports/MSP", _to: "airports/BOI" },
                 ],
             })
-            .map((x) => x.view(["_from", "_to", "ArrTimeUTC"])),
+            .map((x) => x.view(["_from", "_to", "ArrTimeUTC"]))
+            .sort((x, y) => (x.ArrTimeUTC > y.ArrTimeUTC ? 1 : x.ArrTimeUTC == y.ArrTimeUTC ? 0 : -1)),
         [
             {
                 _from: "airports/BOI",
@@ -333,7 +334,7 @@ function query3(qyDB) {
                 _to: "airports/BOI",
                 ArrTimeUTC: "2008-01-08T06:29:00.000Z",
             },
-        ],
+        ].sort((x, y) => (x.ArrTimeUTC > y.ArrTimeUTC ? 1 : x.ArrTimeUTC == y.ArrTimeUTC ? 0 : -1)),
     );
 
     console.timeEnd("query3");
@@ -350,7 +351,7 @@ function query3(qyDB) {
 
 function query4(qyDB) {
     console.time("query4");
-    assert.deepEqual(
+    deepEqual(
         qyDB.queryFiles(["flights", "*", 1, 5], { FlightNum: 860 }).map((x) => x.view(["_from", "_to"])),
         [
             { _from: "airports/IAD", _to: "airports/BOS" },
@@ -365,10 +366,10 @@ function query4(qyDB) {
 
 function query5(qyDB) {
     console.time("query5");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles("**", { vip: true })
-            .map((x) => x.fileContent.name)
+            .map((x) => x.prop("name"))
             .sort(),
         [
             "Amarillo International",
@@ -390,7 +391,7 @@ function query5(qyDB) {
 
 function query6(qyDB) {
     console.time("query6");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles("flights/**", {
                 FlightNum: { $or: [859, 860] },
@@ -407,25 +408,26 @@ function query6(qyDB) {
 
 function query7(qyDB) {
     console.time("query7");
-    assert.deepEqual(
+    deepEqual(
         qyDB
             .queryFiles(["flights", "*", { $or: [1] }, [5, 6, 7]], {
                 $or: [{ _from: "airports/BIS" }, { _to: "airports/BIS" }],
             })
-            .map((x) => x.fileContent.FlightNum),
+            .map((x) => x.prop("FlightNum"))
+            .sort((x, y) => x - y),
         [
             6635, 646, 6541, 5651, 6541, 5648, 687, 309, 6678, 6678, 6636, 439, 1567, 6635, 5647, 6541, 5660, 6541,
             6553, 687, 6553, 309, 5649, 5650, 6636, 439, 6635, 574, 6541, 5651, 5648, 6541, 1705, 1535, 5649, 6678,
             5650, 6636, 6678, 1087,
-        ],
+        ].sort((x, y) => x - y),
     );
     console.timeEnd("query7");
 }
 
 function query8(qyDB) {
     console.time("query8");
-    assert.deepEqual(
-        qyDB.queryFiles("**", { name: "Thigpen " }).map((x) => x.fileContent.name),
+    deepEqual(
+        qyDB.queryFiles("**", { name: "Thigpen " }).map((x) => x.prop("name")),
         ["Thigpen "],
     );
     console.timeEnd("query8");
@@ -450,4 +452,4 @@ function runQueries(qyDB) {
     query8(qyDB);
 }
 
-Object.assign(module.exports, { runQueries });
+export { runQueries };