@opengis/fastify-table 2.0.110 → 2.0.112

package/dist/server/plugins/redis/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../server/plugins/redis/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAW/C,iBAAe,MAAM,CAAC,OAAO,EAAE,eAAe,iBAE7C;AAED,eAAe,MAAM,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../server/plugins/redis/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAa/C,iBAAe,MAAM,CAAC,OAAO,EAAE,eAAe,iBAE7C;AAED,eAAe,MAAM,CAAC"}

package/dist/server/plugins/redis/index.js

@@ -1,6 +1,8 @@
 import redisClients from "./funcs/redisClients.js";
 function close() {
-    Object.keys(redisClients).forEach((key) => {
+    Object.keys(redisClients)
+        .filter((key) => redisClients[key]?.status !== "end")
+        .forEach((key) => {
         console.log("Closing redis client", key);
         redisClients[key].quit();
     });

package/dist/server/routes/file/controllers/export.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../server/routes/file/controllers/export.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AA0B5C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAKzD;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,wBAA8B,WAAW,CACvC,EACE,EAAqB,EACrB,OAAO,EACP,IAAI,EACJ,OAAO,EAAE,QAAQ,EACjB,GAAG,EACH,KAAU,EACV,IAAkB,EAClB,QAAQ,EACR,UAAU,GACX,EAAE;IACD,EAAE,EAAE,UAAU,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,EACD,KAAK,EAAE,YAAY,gBA8WpB"}
+{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../server/routes/file/controllers/export.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAgCzD;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,wBAA8B,WAAW,CACvC,EACE,EAAqB,EACrB,OAAO,EACP,IAAI,EACJ,OAAO,EAAE,QAAQ,EACjB,GAAG,EACH,KAAU,EACV,IAAkB,EAClB,QAAQ,EACR,UAAU,GACX,EAAE;IACD,EAAE,EAAE,UAAU,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,EACD,KAAK,EAAE,YAAY,gBAsYpB"}

package/dist/server/routes/file/controllers/export.js

@@ -6,7 +6,7 @@
 import path from "node:path";
 import { createHash } from "node:crypto";
 import { existsSync } from "node:fs";
-import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import config from "../../../../config.js";
 import logger from "../../../plugins/logger/getLogger.js";
 import getTemplate from "../../../plugins/table/funcs/getTemplate.js";
@@ -19,6 +19,8 @@ import metaFormat from "../../../plugins/table/funcs/metaFormat/index.js";
 import jsonToXls from "./utils/jsonToXls.js";
 import jsonToCsv from "./utils/jsonToCsv.js";
 import formatResult from "./utils/formatResult.js";
+import jsonlToJsonFile from "./utils/jsonlToJsonFile.js";
+import pubsub from "./utils/pubsub.js";
 const startStreamWithTotal = 10000;
 const rootDir = getFolder(config, "local");
 /**
@@ -83,13 +85,6 @@ export default async function exportTable({ pg = pgClients.client, headers, user
             reply,
         });
     }
-    // delete old file, prevent append
-    if (nocache || config.disableCache) {
-        if (cacheFile)
-            await rm(filePath);
-        if (cacheFileJSON && format !== "json")
-            await rm(filePathJSON);
-    }
     const loadTable = await getTemplate("table", table);
     const meta = await getMeta({ pg, table: loadTable?.table || table });
     const viewSql = await getTemplate("view", loadTable?.table || table);
@@ -165,12 +160,22 @@ export default async function exportTable({ pg = pgClients.client, headers, user
             .split(".")
             .pop()}` // check for json data
         : el.name);
-    const send = ((!slice && +filtered > startStreamWithTotal) || stream) &&
-        process.env.NODE_ENV !== "test"
-        ? eventStream(reply)
-        : process.env.NODE_ENV === "test"
-            ? console.log
-            : () => { };
+    const isStream = ((!slice && +filtered > startStreamWithTotal) || stream) &&
+        process.env.NODE_ENV !== "test" &&
+        !process.env.VITEST;
+    const sendOriginal = isStream ? eventStream(reply) : () => { };
+    const redisKey = `exportTable:${fileName}`;
+    const send = isStream
+        ? await pubsub(redisKey, (msg, finishStream = false) => {
+            sendOriginal(msg, finishStream);
+        }, query.reload && user?.user_type?.includes?.("admin"))
+        : () => { };
+    // delete old file, prevent append
+    if (cacheFile && (nocache || config.disableCache)) {
+        await rm(filePath);
+    }
+    // create directory if not exists
+    await mkdir(path.dirname(filePath), { recursive: true });
     // export xlsx / csv / json
     const source = loadTable?.title || loadTable?.ua || table || sourceName;
     const interval = setInterval(async () => {
@@ -183,6 +188,19 @@ export default async function exportTable({ pg = pgClients.client, headers, user
     let seq = 0;
     send(`Всього в реєстрі: ${result.total} (${filtered} з урахуванням фільтрів)`);
     if (!cacheFileJSON || nocache || config.disableCache) {
+        // delete old json line file, prevent append
+        if (existsSync(filePathJSON.replace(/.json$/, ".jsonl"))) {
+            await rm(filePathJSON.replace(/.json$/, ".jsonl"), {
+                recursive: true,
+                force: true,
+            });
+        }
+        // delete old file, prevent append
+        if (existsSync(filePathJSON)) {
+            await rm(filePathJSON, { recursive: true, force: true });
+        }
+        // create empty temporary json lines file to append to
+        await writeFile(filePathJSON.replace(/.json$/, ".jsonl"), "");
         while (+filtered - offset > 0 && !res?.error) {
             try {
                 send(`Оброблено: ${offset}/${filtered}`);
@@ -211,19 +229,8 @@ export default async function exportTable({ pg = pgClients.client, headers, user
                             .forEach((key) => delete row[key]);
                     });
                 }
-                const jsonFileExists = existsSync(filePathJSON);
-                // convert from json to format
-                if (!jsonFileExists) {
-                    // if json not exists
-                    await mkdir(path.dirname(filePath), { recursive: true });
-                    await writeFile(filePathJSON, JSON.stringify(rows));
-                }
-                else {
-                    // if json exists
-                    const jsonData = JSON.parse((await readFile(filePathJSON, "utf8")) || "{}");
-                    const moreData = jsonData.concat(rows); // rewrite to appendFile?
-                    await writeFile(filePathJSON, JSON.stringify(moreData));
-                }
+                const data = rows.map((row) => JSON.stringify(row)).join("\n") + "\n";
+                await appendFile(filePathJSON.replace(/.json$/, ".jsonl"), data);
                 offset += rows.length;
                 page++;
             }
@@ -242,9 +249,12 @@ export default async function exportTable({ pg = pgClients.client, headers, user
             }
         }
     }
+    // convert json line to json array
+    await jsonlToJsonFile(filePathJSON.replace(/.json$/, ".jsonl"));
     clearInterval(interval);
     if (res.error) {
-        send(res.error, 1);
+        send("finish");
+        sendOriginal(res.error, 1);
         return reply.status(500).send(res.error);
     }
     logger.file("export/table", {
@@ -298,7 +308,8 @@ export default async function exportTable({ pg = pgClients.client, headers, user
     if (resp.error) {
         return reply.status(resp.status || 500).send(resp.error);
     }
-    send("Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних", 1);
+    send("finish");
+    sendOriginal("Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних", 1);
     return formatResult({
         filePath,
         formatAnswer,

package/dist/server/routes/file/controllers/resize.d.ts

@@ -11,6 +11,7 @@ export default function resize({ query, }: {
         h?: string;
         nocache?: any;
         format?: string;
+        noimage?: any;
     };
 }, reply: FastifyReply): Promise<never>;
 //# sourceMappingURL=resize.d.ts.map

package/dist/server/routes/file/controllers/resize.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"resize.d.ts","sourceRoot":"","sources":["../../../../../server/routes/file/controllers/resize.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AA+B5C;;GAEG;AAEH,wBAA8B,MAAM,CAClC,EACE,KAAK,GACN,EAAE;IACD,KAAK,EAAE;QACL,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAC1B,IAAI,EAAE,MAAM,CAAC;QACb,CAAC,CAAC,EAAE,MAAM,CAAC;QACX,CAAC,CAAC,EAAE,MAAM,CAAC;QACX,OAAO,CAAC,EAAE,GAAG,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH,EACD,KAAK,EAAE,YAAY,kBAmIpB"}
+{"version":3,"file":"resize.d.ts","sourceRoot":"","sources":["../../../../../server/routes/file/controllers/resize.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAoC5C;;GAEG;AAEH,wBAA8B,MAAM,CAClC,EACE,KAAK,GACN,EAAE;IACD,KAAK,EAAE;QACL,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAC1B,IAAI,EAAE,MAAM,CAAC;QACb,CAAC,CAAC,EAAE,MAAM,CAAC;QACX,CAAC,CAAC,EAAE,MAAM,CAAC;QACX,OAAO,CAAC,EAAE,GAAG,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,GAAG,CAAC;KACf,CAAC;CACH,EACD,KAAK,EAAE,YAAY,kBAwIpB"}

package/dist/server/routes/file/controllers/resize.js

@@ -15,11 +15,15 @@ const getHeight = (width, size, ratio = 1) => {
     return defaultHeight;
 };
 const { resizeImage } = grpc();
+const transparentGif = Buffer.from("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "base64");
 /**
  * Апі використовується для зміни розміру фото за шляхом
  */
 export default async function resize({ query, }, reply) {
-    const { filepath, quality, size, w, h, nocache, format } = query || {};
+    const { filepath, quality, size, w, h, nocache, format, noimage } = query || {};
+    if (noimage) {
+        return reply.headers({ "Content-Type": "image/gif" }).send(transparentGif);
+    }
     if (!filepath) {
         return reply
             .status(400)

package/dist/server/routes/file/controllers/utils/jsonlToJsonFile.d.ts

@@ -0,0 +1,2 @@
+export default function jsonlToJsonFile(inputPath: string): Promise<string>;
+//# sourceMappingURL=jsonlToJsonFile.d.ts.map

package/dist/server/routes/file/controllers/utils/jsonlToJsonFile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonlToJsonFile.d.ts","sourceRoot":"","sources":["../../../../../../server/routes/file/controllers/utils/jsonlToJsonFile.ts"],"names":[],"mappings":"AAKA,wBAA8B,eAAe,CAAC,SAAS,EAAE,MAAM,mBAiC9D"}

package/dist/server/routes/file/controllers/utils/jsonlToJsonFile.js

@@ -0,0 +1,32 @@
+import { once } from "node:events";
+import { rm } from "node:fs/promises";
+import { createInterface } from "node:readline";
+import { createReadStream, createWriteStream } from "node:fs";
+export default async function jsonlToJsonFile(inputPath) {
+    const outputPath = inputPath.replace(/.jsonl$/g, ".json");
+    const input = createReadStream(inputPath);
+    const output = createWriteStream(outputPath);
+    const stream = createInterface({ input, crlfDelay: Infinity });
+    output.write("[\n");
+    let first = true;
+    stream.on("line", (line) => {
+        if (!line.trim())
+            return;
+        if (!first)
+            output.write(",\n");
+        output.write(JSON.stringify(JSON.parse(line)));
+        first = false;
+    });
+    // await stream to close /until convert finished
+    await once(stream, "close");
+    output.write("\n]");
+    output.end();
+    // wait for write stream to finish flushing
+    await once(output, "finish");
+    // rm temporary json line
+    await rm(inputPath, {
+        recursive: true,
+        force: true,
+    });
+    return outputPath;
+}

package/dist/server/routes/file/controllers/utils/pubsub.d.ts

@@ -0,0 +1,2 @@
+export default function subscribe(code: string, cb: any, reload: any): Promise<any>;
+//# sourceMappingURL=pubsub.d.ts.map

package/dist/server/routes/file/controllers/utils/pubsub.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pubsub.d.ts","sourceRoot":"","sources":["../../../../../../server/routes/file/controllers/utils/pubsub.ts"],"names":[],"mappings":"AAkDA,wBAA8B,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,gBA2EzE"}

package/dist/server/routes/file/controllers/utils/pubsub.js

@@ -0,0 +1,112 @@
+const keyCacheSubscribe = "emit_cache";
+const listenter = {}; // callback send message listener
+const promise = {}; // promise await for subscribe
+const resolve = {}; // resolve
+const publish = {}; // publish
+import config from "../../../../../config.js";
+import getRedis from "../../../../plugins/redis/funcs/getRedis.js";
+// getRedisPublisher
+function getRedisPublisher() {
+    const rst = getRedis({
+        name: "rst",
+        ...(config.redis || {}),
+    });
+    const subscriber = getRedis({
+        name: "sub",
+        ...(config.redis || {}),
+    });
+    subscriber.unsubscribe(keyCacheSubscribe);
+    subscriber.subscribe(keyCacheSubscribe, () => { });
+    subscriber.on("message", async (channel, text) => {
+        const [code, msg] = text.split("|||");
+        if (msg === "finish" && resolve[code]) {
+            resolve[code]();
+            resolve[code] = null;
+        }
+        listenter[code].forEach((cb1, i) => {
+            cb1(`${msg} - ${i}`);
+        });
+    });
+    return rst;
+}
+// native publisher
+function getPublisher() {
+    const rst = {
+        publish(key, text) {
+            const [code, msg] = text.split("|||");
+            listenter[code]?.forEach((cb1, i) => {
+                cb1?.(`${msg} - ${i}`);
+            });
+        },
+    };
+    return rst;
+}
+// !!! work non cluster mode for cluster subscribe redis
+export default async function subscribe(code, cb, reload) {
+    try {
+        const rclient = getRedis();
+        const { isRedis, timeout = 150 } = config?.redis?.pubsub || {}; // test!
+        const publisher = isRedis ? getRedisPublisher() : getPublisher();
+        const keyCacheCode = `${keyCacheSubscribe}:${code}`;
+        // check code is run
+        const isNotRun = await rclient.setnx(keyCacheCode, 1);
+        const ttl = await rclient.ttl(keyCacheCode);
+        // if Run
+        if (!isNotRun && !reload && ttl > 0) {
+            // add listener
+            cb(`==== run as slave  ttl: ${ttl}==== `);
+            const alldata = await rclient.lrange(keyCacheCode + ":list", 0, -1);
+            alldata.forEach((el) => {
+                cb(el);
+            });
+            // for redis
+            if (!listenter[code]) {
+                listenter[code] = [];
+                promise[code] = new Promise((rsv) => {
+                    resolve[code] = rsv;
+                });
+            }
+            // check master finish
+            const interval = setInterval(async () => {
+                const ttl1 = await rclient.ttl(keyCacheCode);
+                if (ttl1 < 0) {
+                    clearInterval(interval);
+                    cb(`finish master timeout ${timeout}`, 1);
+                }
+            }, 5000);
+            listenter[code].push(cb);
+            return promise[code];
+        }
+        await rclient.expire(keyCacheCode, timeout);
+        // if not run
+        // create return promise
+        promise[code] = new Promise((rsv) => {
+            resolve[code] = rsv;
+        });
+        listenter[code] = [cb];
+        // listenter[code].push(cb);
+        // create publish function
+        publish[code] = (msg) => {
+            rclient.rpush(keyCacheCode + ":list", msg);
+            rclient.expire(keyCacheCode, timeout);
+            // send message to all listener
+            publisher.publish(keyCacheSubscribe, `${code}|||${msg}`); // redis
+            // finish and clear
+            if (msg === "finish") {
+                // clear code to run again
+                rclient.del(keyCacheCode);
+                rclient.del(keyCacheCode + ":list");
+                // resolve promise
+                resolve[code]();
+                // clear
+                resolve[code] = null;
+                promise[code] = null;
+                listenter[code] = [];
+            }
+        };
+        return publish[code];
+    }
+    catch (err) {
+        console.error(err.toString());
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengis/fastify-table",
-  "version": "2.0.110",
+  "version": "2.0.112",
   "type": "module",
   "description": "core-plugins",
   "keywords": [