@saltcorn/qlik-qvd 0.1.0 → 0.1.2

package/.claude/settings.local.json

@@ -0,0 +1,7 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(node -c index.js)"
+    ]
+  }
+}

package/README.md

@@ -0,0 +1,11 @@
+# qlik-qvd
+
+Provides the import_qvd_file function, which takes as argumnets:
+
+- a filename (a file in the file store) 
+- optionally a table name (otherwise it will use the name found in the QVD file)
+
+and when run:
+
+1. If the table does not exist, creates it with the fields in the QVD file
+2. Imports the rows in the QVD file into the table

package/index.js

@@ -2,17 +2,89 @@ const File = require("@saltcorn/data/models/file");
 const Table = require("@saltcorn/data/models/table");
 const Field = require("@saltcorn/data/models/field");
 const { getState } = require("@saltcorn/data/db/state");
+const db = require("@saltcorn/data/db");
+
+const { Readable } = require("stream");
 
 const { QvdDataFrame, QvdFileReader } = require("qvd4js");
 
-const numberFormatToType = (nf) => {
-  if (nf.Type === "REAL") return "Float";
-  if (nf.Type === "INTEGER") return "Integer";
-  if (nf.Type === "TIME") return "String";
-  if (nf.Type === "UNKNOWN") return "String";
-  throw new Error("Unknown NumberFormat: " + JSON.stringify(nf));
+// Serialise a single value into a CSV cell suitable for a Postgres
+// `COPY ... FROM STDIN CSV HEADER` ingest. An empty cell becomes NULL.
+const csvCell = (v) => {
+  if (v === null || v === undefined) return "";
+  let s;
+  if (v instanceof Date) s = v.toISOString();
+  else s = String(v);
+  if (/[",\n\r]/.test(s)) return '"' + s.replace(/"/g, '""') + '"';
+  return s;
+};
+
+// Resolve a single QVD symbol to the value qvd4js would actually place in the
+// data frame: QvdSymbol.toPrimaryValue() prioritises the string representation,
+// and QvdFileReader.load() then coerces numeric-looking strings to numbers.
+const symbolValue = (symbol) => {
+  const value = symbol.toPrimaryValue();
+  if (typeof value === "string" && value !== "" && !isNaN(Number(value)))
+    return Number(value);
+  return value;
+};
+
+// Inspect every distinct symbol of a field to discover what it really holds.
+// The QVD NumberFormat cannot be trusted on its own: a field tagged INTEGER may
+// still contain pure strings (e.g. "KBS108244839"), so the symbol table is the
+// authoritative source for the column type.
+const scanSymbols = (symbols) => {
+  let hasString = false;
+  let hasNumber = false;
+  let hasFloat = false;
+  for (const symbol of symbols || []) {
+    const value = symbolValue(symbol);
+    if (value === null || value === undefined) continue;
+    if (typeof value === "number") {
+      hasNumber = true;
+      if (!Number.isInteger(value)) hasFloat = true;
+    } else {
+      hasString = true;
+    }
+  }
+  return { hasString, hasNumber, hasFloat };
+};
+
+const deduceFieldType = (field, symbols) => {
+  const nf = field.NumberFormat;
+  // Dates/times are identified through the number format only; in the symbol
+  // table they appear as numeric serials or formatted strings.
+  if (nf.Type === "TIME") return { type: "String" };
+  if (nf.Type === "DATE")
+    return {
+      type: "Date",
+      attributes: nf.Fmt === "DD.MM.YYYY" ? { day_only: true } : {},
+    };
+  if (nf.Type === "TIMESTAMP")
+    return {
+      type: "Date",
+    };
+
+  // For everything else, let the actual symbol contents decide between String,
+  // Float and Integer rather than relying on the (unreliable) NumberFormat.
+  const { hasString, hasNumber, hasFloat } = scanSymbols(symbols);
+  if (hasString) return { type: "String" };
+  if (hasFloat || nf.Type === "REAL" || nf.Type === "FIX")
+    return { type: "Float" };
+  if (hasNumber || nf.Type === "INTEGER") return { type: "Integer" };
+  // No usable symbols (e.g. an all-null column): fall back to String.
+  return { type: "String" };
 };
 
+const QLIK_EPOCH_MS = Date.UTC(1899, 11, 30); // 1899-12-30 00:00:00 (month is 0-indexed)
+const MS_PER_DAY = 86400000;
+
+function fromQlik(serial, { roundToSeconds = true } = {}) {
+  let ms = QLIK_EPOCH_MS + serial * MS_PER_DAY;
+  if (roundToSeconds) ms = Math.round(ms / 1000) * 1000;
+  return new Date(ms);
+}
+
 module.exports = {
   sc_plugin_api_version: 1,
   plugin_name: "qlik-qvd",
@@ -24,35 +96,82 @@ module.exports = {
         const reader = new QvdFileReader(file.location);
         const df = await reader.load();
         const qvdTableName = reader._header.QvdTableHeader.TableName;
-        const fields =
+        let fields =
           reader._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
+        // A QVD with a single field is parsed as an object, not an array.
+        // Normalise so it lines up with reader._symbolTable, which is always an
+        // array indexed by field position.
+        if (!Array.isArray(fields)) fields = [fields];
 
         let table = Table.findOne({ name: table_name || qvdTableName });
         let field_names = [];
         if (!table) {
           table = await Table.create(table_name || qvdTableName);
 
-          for (const field of fields) {
+          for (let i = 0; i < fields.length; i++) {
+            const field = fields[i];
             const fld = {
               table,
               label: field.FieldName,
-              type: numberFormatToType(field.NumberFormat),
-              attributes: {},
+              ...deduceFieldType(field, reader._symbolTable[i]),
             };
 
             const f = await Field.create(fld);
             field_names.push(f.name);
           }
-          await getState.refresh_tables()
+          await getState().refresh_tables();
         } else {
           field_names = fields.map((f) => Field.labelToName(f.FieldName));
         }
-        for (const row of df.data) {
-          const o = {};
-          for (let index = 0; index < field_names.length; index++) {
-            o[field_names[index]] = row[index];
+        const timeStampFields = new Set(
+          table.fields
+            .filter((f) => f.type?.name === "Date" && !f.attributes?.day_only)
+            .map((f) => f.name),
+        );
+
+        // Convert the raw QVD value for a given target field to the value we
+        // want stored, applying the Qlik-serial -> Date conversion for
+        // timestamp fields (mirrors the row-by-row path below).
+        const convertValue = (fname, val) => {
+          if (timeStampFields.has(fname) && typeof val === "number")
+            return fromQlik(val);
+          return val;
+        };
+
+        // Fast path: bulk-load via Postgres COPY when the driver supports it.
+        // db.copyFrom is only defined on the Postgres driver, not SQLite.
+        if (db.copyFrom) {
+          // Stream a CSV of the QVD data, using the plugin's field names as the
+          // header, instead of issuing one insertRow per row.
+          const csvStream = Readable.from(
+            (function* () {
+              yield field_names.map(csvCell).join(",") + "\n";
+              for (const row of df.data) {
+                const cells = new Array(field_names.length);
+                for (let index = 0; index < field_names.length; index++) {
+                  const fname = field_names[index];
+                  cells[index] = csvCell(convertValue(fname, row[index]));
+                }
+                yield cells.join(",") + "\n";
+              }
+            })(),
+          );
+
+          const client = await db.getClient();
+          try {
+            await db.copyFrom(csvStream, table.name, field_names, client);
+          } finally {
+            await client.release(true);
+          }
+        } else {
+          for (const row of df.data) {
+            const o = {};
+            for (let index = 0; index < field_names.length; index++) {
+              const fname = field_names[index];
+              o[fname] = convertValue(fname, row[index]);
+            }
+            await table.insertRow(o);
           }
-          await table.insertRow(o);
         }
       },
       description: "Convert a list of JSON objects to a CSV string",

package/package.json

@@ -1,12 +1,11 @@
 {
   "name": "@saltcorn/qlik-qvd",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Interacting with Qlik QVD files",
   "main": "index.js",
   "dependencies": {
     "@saltcorn/markup": "^0.8.0",
     "@saltcorn/data": "^0.8.0",
-    "qvdrs": "0.7.0",
     "qvd4js":"1.0.5"
   },
   "author": "Tom Nielsen",