latticesql 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/cli.js +1899 -135
- package/dist/index.cjs +1948 -153
- package/dist/index.d.cts +360 -1
- package/dist/index.d.ts +360 -1
- package/dist/index.js +1908 -127
- package/docs/api-reference.md +60 -4
- package/docs/architecture.md +24 -0
- package/docs/assistant.md +23 -0
- package/docs/examples/dashboard.html +284 -0
- package/docs/importing.md +118 -0
- package/docs/retrieval.md +31 -0
- package/package.json +7 -3
package/dist/index.js
CHANGED
|
@@ -5550,7 +5550,9 @@ function withCredentialLock(fn) {
|
|
|
5550
5550
|
fd = openSync2(lockPath, "wx");
|
|
5551
5551
|
break;
|
|
5552
5552
|
} catch (err) {
|
|
5553
|
-
|
|
5553
|
+
const code = err.code;
|
|
5554
|
+
const contended = code === "EEXIST" || process.platform === "win32" && (code === "EPERM" || code === "EACCES");
|
|
5555
|
+
if (!contended) throw err;
|
|
5554
5556
|
try {
|
|
5555
5557
|
if (Date.now() - statSync4(lockPath).mtimeMs > LOCK_STALE_MS) {
|
|
5556
5558
|
unlinkSync3(lockPath);
|
|
@@ -6436,6 +6438,7 @@ function deriveCanonicalContexts(tables) {
|
|
|
6436
6438
|
childrenOf.set(rel.table, list);
|
|
6437
6439
|
}
|
|
6438
6440
|
}
|
|
6441
|
+
const byName = new Map(tables.map((t8) => [t8.name, t8.definition]));
|
|
6439
6442
|
const out = [];
|
|
6440
6443
|
for (const { name, definition } of tables) {
|
|
6441
6444
|
const files = {};
|
|
@@ -6451,11 +6454,32 @@ function deriveCanonicalContexts(tables) {
|
|
|
6451
6454
|
};
|
|
6452
6455
|
}
|
|
6453
6456
|
for (const child of childrenOf.get(name) ?? []) {
|
|
6454
|
-
|
|
6455
|
-
|
|
6456
|
-
|
|
6457
|
-
|
|
6458
|
-
|
|
6457
|
+
const childDef = byName.get(child.table);
|
|
6458
|
+
const childBt = childDef ? belongsToRelations(childDef) : [];
|
|
6459
|
+
const [rel0, rel1] = childBt;
|
|
6460
|
+
if (childDef && rel0 && rel1 && isRenderJunction(childDef, childBt)) {
|
|
6461
|
+
const localRel = rel0.foreignKey === child.foreignKey ? rel0 : rel1;
|
|
6462
|
+
const remoteRel = localRel === rel0 ? rel1 : rel0;
|
|
6463
|
+
const fileKey = remoteRel.table === name ? `${child.table.toUpperCase()}__${remoteRel.foreignKey.toUpperCase()}.md` : `${remoteRel.table.toUpperCase()}.md`;
|
|
6464
|
+
files[fileKey] = {
|
|
6465
|
+
source: {
|
|
6466
|
+
type: "manyToMany",
|
|
6467
|
+
junctionTable: child.table,
|
|
6468
|
+
localKey: localRel.foreignKey,
|
|
6469
|
+
remoteKey: remoteRel.foreignKey,
|
|
6470
|
+
remoteTable: remoteRel.table,
|
|
6471
|
+
references: remoteRel.references ?? "id"
|
|
6472
|
+
},
|
|
6473
|
+
render: renderRelated(remoteRel.table),
|
|
6474
|
+
omitIfEmpty: true
|
|
6475
|
+
};
|
|
6476
|
+
} else {
|
|
6477
|
+
files[`${child.table.toUpperCase()}.md`] = {
|
|
6478
|
+
source: { type: "hasMany", table: child.table, foreignKey: child.foreignKey },
|
|
6479
|
+
render: renderRelated(child.table),
|
|
6480
|
+
omitIfEmpty: true
|
|
6481
|
+
};
|
|
6482
|
+
}
|
|
6459
6483
|
}
|
|
6460
6484
|
out.push({
|
|
6461
6485
|
table: name,
|
|
@@ -6468,6 +6492,15 @@ function deriveCanonicalContexts(tables) {
|
|
|
6468
6492
|
}
|
|
6469
6493
|
return out;
|
|
6470
6494
|
}
|
|
6495
|
+
function isRenderJunction(def, bt) {
|
|
6496
|
+
if (bt.length !== 2) return false;
|
|
6497
|
+
const fks = new Set(bt.map((r6) => r6.foreignKey));
|
|
6498
|
+
if (fks.size !== 2) return false;
|
|
6499
|
+
const pk = Array.isArray(def.primaryKey) ? def.primaryKey : def.primaryKey != null ? [def.primaryKey] : [];
|
|
6500
|
+
if (pk.length === 2 && pk.every((c6) => fks.has(c6))) return true;
|
|
6501
|
+
const SYSTEM2 = /* @__PURE__ */ new Set(["id", "created_at", "updated_at", "deleted_at"]);
|
|
6502
|
+
return Object.keys(def.columns).every((c6) => fks.has(c6) || SYSTEM2.has(c6));
|
|
6503
|
+
}
|
|
6471
6504
|
function belongsToRelations(def) {
|
|
6472
6505
|
return Object.values(def.relations ?? {}).filter(
|
|
6473
6506
|
(r6) => r6.type === "belongsTo"
|
|
@@ -6919,6 +6952,19 @@ var init_vector_index = __esm({
|
|
|
6919
6952
|
}
|
|
6920
6953
|
});
|
|
6921
6954
|
|
|
6955
|
+
// src/search/limits.ts
|
|
6956
|
+
function clampTopK(topK) {
|
|
6957
|
+
if (!Number.isFinite(topK)) return 1;
|
|
6958
|
+
return Math.min(Math.max(1, Math.floor(topK)), SEARCH_TOPK_MAX);
|
|
6959
|
+
}
|
|
6960
|
+
var SEARCH_TOPK_MAX;
|
|
6961
|
+
var init_limits = __esm({
|
|
6962
|
+
"src/search/limits.ts"() {
|
|
6963
|
+
"use strict";
|
|
6964
|
+
SEARCH_TOPK_MAX = 1e3;
|
|
6965
|
+
}
|
|
6966
|
+
});
|
|
6967
|
+
|
|
6922
6968
|
// src/search/embeddings.ts
|
|
6923
6969
|
async function ensureEmbeddingsTable(adapter) {
|
|
6924
6970
|
let cols = [];
|
|
@@ -7065,9 +7111,10 @@ function cosineSimilarity(a6, b6) {
|
|
|
7065
7111
|
}
|
|
7066
7112
|
async function searchByEmbedding(adapter, table, queryText, config, topK, minScore, pkColumn = "id") {
|
|
7067
7113
|
const queryVector = await config.embed(queryText);
|
|
7114
|
+
const k6 = clampTopK(topK);
|
|
7068
7115
|
let ranked;
|
|
7069
7116
|
if (await vectorIndexAvailable(adapter) && await hasVectorIndex(adapter, table)) {
|
|
7070
|
-
const hits = await searchVectorIndex(adapter, table, queryVector,
|
|
7117
|
+
const hits = await searchVectorIndex(adapter, table, queryVector, k6 * 4, minScore);
|
|
7071
7118
|
ranked = hits.map((h6) => ({
|
|
7072
7119
|
pk: h6.pk,
|
|
7073
7120
|
score: h6.score,
|
|
@@ -7075,7 +7122,7 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
7075
7122
|
content: h6.content
|
|
7076
7123
|
}));
|
|
7077
7124
|
} else {
|
|
7078
|
-
ranked = await scanChunks(adapter, table, queryVector, minScore);
|
|
7125
|
+
ranked = await scanChunks(adapter, table, queryVector, minScore, config.maxScanChunks);
|
|
7079
7126
|
}
|
|
7080
7127
|
const bestByRow = /* @__PURE__ */ new Map();
|
|
7081
7128
|
for (const r6 of ranked) {
|
|
@@ -7100,11 +7147,20 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
7100
7147
|
if (r6.content !== null) result.matchedContent = r6.content;
|
|
7101
7148
|
}
|
|
7102
7149
|
results.push(result);
|
|
7103
|
-
if (results.length >=
|
|
7150
|
+
if (results.length >= k6) break;
|
|
7104
7151
|
}
|
|
7105
7152
|
return results;
|
|
7106
7153
|
}
|
|
7107
|
-
async function scanChunks(adapter, table, queryVector, minScore) {
|
|
7154
|
+
async function scanChunks(adapter, table, queryVector, minScore, maxScanChunks) {
|
|
7155
|
+
if (maxScanChunks !== void 0) {
|
|
7156
|
+
const countRows = await allAsyncOrSync(
|
|
7157
|
+
adapter,
|
|
7158
|
+
`SELECT COUNT(*) AS n FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
7159
|
+
[table]
|
|
7160
|
+
);
|
|
7161
|
+
const n3 = Number(countRows[0]?.n ?? 0);
|
|
7162
|
+
if (n3 > maxScanChunks) throw new EmbeddingScanTooLargeError(table, n3, maxScanChunks);
|
|
7163
|
+
}
|
|
7108
7164
|
const stored = await allAsyncOrSync(
|
|
7109
7165
|
adapter,
|
|
7110
7166
|
`SELECT "row_pk", "chunk_index", "content", "embedding", "vec_dim" FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
@@ -7214,13 +7270,14 @@ async function refreshEmbeddings(adapter, table, config, pkColumn = "id", opts =
|
|
|
7214
7270
|
}
|
|
7215
7271
|
return { embedded, skipped, removed };
|
|
7216
7272
|
}
|
|
7217
|
-
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError;
|
|
7273
|
+
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError, EmbeddingScanTooLargeError;
|
|
7218
7274
|
var init_embeddings = __esm({
|
|
7219
7275
|
"src/search/embeddings.ts"() {
|
|
7220
7276
|
"use strict";
|
|
7221
7277
|
init_adapter();
|
|
7222
7278
|
init_chunking();
|
|
7223
7279
|
init_vector_index();
|
|
7280
|
+
init_limits();
|
|
7224
7281
|
EMBEDDINGS_TABLE = "_lattice_embeddings";
|
|
7225
7282
|
EmbeddingDimensionMismatchError = class extends Error {
|
|
7226
7283
|
constructor(table, expected, found) {
|
|
@@ -7233,6 +7290,17 @@ var init_embeddings = __esm({
|
|
|
7233
7290
|
this.name = "EmbeddingDimensionMismatchError";
|
|
7234
7291
|
}
|
|
7235
7292
|
};
|
|
7293
|
+
EmbeddingScanTooLargeError = class extends Error {
|
|
7294
|
+
constructor(table, found, limit) {
|
|
7295
|
+
super(
|
|
7296
|
+
`Embedding scan on "${table}" would read ${String(found)} stored chunk vectors, over the configured maxScanChunks of ${String(limit)}. Add a native vector index (pgvector) for this table or raise maxScanChunks \u2014 Lattice will not silently truncate the scan, which would return incomplete results.`
|
|
7297
|
+
);
|
|
7298
|
+
this.table = table;
|
|
7299
|
+
this.found = found;
|
|
7300
|
+
this.limit = limit;
|
|
7301
|
+
this.name = "EmbeddingScanTooLargeError";
|
|
7302
|
+
}
|
|
7303
|
+
};
|
|
7236
7304
|
}
|
|
7237
7305
|
});
|
|
7238
7306
|
|
|
@@ -7585,7 +7653,7 @@ async function fetchLiveRows2(adapter, table, ids, pkColumn) {
|
|
|
7585
7653
|
return out;
|
|
7586
7654
|
}
|
|
7587
7655
|
async function hybridSearch(adapter, table, query, opts = {}) {
|
|
7588
|
-
const topK = opts.topK ?? 10;
|
|
7656
|
+
const topK = clampTopK(opts.topK ?? 10);
|
|
7589
7657
|
const rrfK = opts.rrfK ?? 60;
|
|
7590
7658
|
const pool = opts.poolSize ?? Math.max(topK * 4, 20);
|
|
7591
7659
|
const pkColumn = opts.pkColumn ?? "id";
|
|
@@ -7686,6 +7754,7 @@ var init_hybrid = __esm({
|
|
|
7686
7754
|
init_fts();
|
|
7687
7755
|
init_ranking();
|
|
7688
7756
|
init_rerank();
|
|
7757
|
+
init_limits();
|
|
7689
7758
|
}
|
|
7690
7759
|
});
|
|
7691
7760
|
|
|
@@ -8413,6 +8482,26 @@ RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
|
8413
8482
|
);
|
|
8414
8483
|
$fn$;
|
|
8415
8484
|
|
|
8485
|
+
-- Delete-event visibility, decided from the PRE-DELETE snapshot the delete trigger
|
|
8486
|
+
-- captures (the live row + its ownership record are gone after a delete, so
|
|
8487
|
+
-- lattice_row_visible can't be used). Keyed on session_user, SECURITY DEFINER \u2014
|
|
8488
|
+
-- the same per-recipient gate. MUST MIRROR lattice_row_visible's rule: the row is
|
|
8489
|
+
-- visible iff this member owned it, OR it was 'everyone', OR it was 'custom' and
|
|
8490
|
+
-- this member was a grantee. A NULL owner snapshot (a legacy delete emitted before
|
|
8491
|
+
-- the snapshot columns, or a row with no ownership record) yields false \u2014 fail
|
|
8492
|
+
-- closed, never forward. (tests/integration assert this agrees with
|
|
8493
|
+
-- lattice_row_visible for all three visibility states \u2014 the no-drift guard.)
|
|
8494
|
+
CREATE OR REPLACE FUNCTION lattice_delete_visible(
|
|
8495
|
+
p_owner_role text, p_visibility text, p_grantees text[]
|
|
8496
|
+
)
|
|
8497
|
+
RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
8498
|
+
SELECT p_owner_role IS NOT NULL AND (
|
|
8499
|
+
p_owner_role = session_user
|
|
8500
|
+
OR p_visibility = 'everyone'
|
|
8501
|
+
OR (p_visibility = 'custom' AND session_user = ANY(COALESCE(p_grantees, ARRAY[]::text[])))
|
|
8502
|
+
);
|
|
8503
|
+
$fn$;
|
|
8504
|
+
|
|
8416
8505
|
-- Shared owner gate: raises unless the connected member owns (p_table, p_pk).
|
|
8417
8506
|
-- p_action is spliced into the message so every caller keeps its exact wording.
|
|
8418
8507
|
-- SECURITY DEFINER + session_user (never current_user), the cloud identity invariant.
|
|
@@ -8587,6 +8676,14 @@ CREATE TABLE IF NOT EXISTS "__lattice_changes" (
|
|
|
8587
8676
|
"created_at" timestamptz NOT NULL DEFAULT now()
|
|
8588
8677
|
);
|
|
8589
8678
|
|
|
8679
|
+
-- Pre-delete visibility snapshot columns (added to existing clouds via ADD COLUMN
|
|
8680
|
+
-- IF NOT EXISTS). A delete event carries the row's visibility AT DELETE TIME so the
|
|
8681
|
+
-- live fan-out can gate it per recipient even though the ownership record is gone.
|
|
8682
|
+
-- NULL on upserts.
|
|
8683
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_owner_role" text;
|
|
8684
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_visibility" text;
|
|
8685
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_grantees" text[];
|
|
8686
|
+
|
|
8590
8687
|
CREATE OR REPLACE FUNCTION lattice_notify_change() RETURNS trigger
|
|
8591
8688
|
LANGUAGE plpgsql AS $fn$
|
|
8592
8689
|
BEGIN
|
|
@@ -8596,7 +8693,10 @@ BEGIN
|
|
|
8596
8693
|
'pk', NEW."pk",
|
|
8597
8694
|
'op', NEW."op",
|
|
8598
8695
|
'owner_role', NEW."owner_role",
|
|
8599
|
-
'created_at', NEW."created_at"
|
|
8696
|
+
'created_at', NEW."created_at",
|
|
8697
|
+
'del_owner_role', NEW."del_owner_role",
|
|
8698
|
+
'del_visibility', NEW."del_visibility",
|
|
8699
|
+
'del_grantees', NEW."del_grantees"
|
|
8600
8700
|
)::text);
|
|
8601
8701
|
RETURN NEW;
|
|
8602
8702
|
END $fn$;
|
|
@@ -8802,10 +8902,22 @@ BEGIN
|
|
|
8802
8902
|
VALUES (${lit}, ${pkNew}, 'upsert', session_user);
|
|
8803
8903
|
RETURN NEW;
|
|
8804
8904
|
ELSIF TG_OP = 'DELETE' THEN
|
|
8905
|
+
-- Snapshot the row's visibility BEFORE the cascade removes its ownership +
|
|
8906
|
+
-- grant records, so the realtime fan-out can gate the delete event per
|
|
8907
|
+
-- recipient (the live predicate can't \u2014 these records are gone post-delete).
|
|
8908
|
+
-- The grantee list is captured here because the grant rows are deleted in the
|
|
8909
|
+
-- same statement below; after that the 'custom' audience is unrecoverable.
|
|
8910
|
+
INSERT INTO "__lattice_changes"
|
|
8911
|
+
("table_name","pk","op","owner_role","del_owner_role","del_visibility","del_grantees")
|
|
8912
|
+
VALUES (${lit}, ${pkOld}, 'delete', session_user,
|
|
8913
|
+
(SELECT o."owner_role" FROM "__lattice_owners" o
|
|
8914
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
8915
|
+
(SELECT o."visibility" FROM "__lattice_owners" o
|
|
8916
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
8917
|
+
COALESCE((SELECT array_agg(g."grantee_role") FROM "__lattice_row_grants" g
|
|
8918
|
+
WHERE g."table_name" = ${lit} AND g."pk" = ${pkOld}), ARRAY[]::text[]));
|
|
8805
8919
|
DELETE FROM "__lattice_owners" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8806
8920
|
DELETE FROM "__lattice_row_grants" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8807
|
-
INSERT INTO "__lattice_changes" ("table_name","pk","op","owner_role")
|
|
8808
|
-
VALUES (${lit}, ${pkOld}, 'delete', session_user);
|
|
8809
8921
|
RETURN OLD;
|
|
8810
8922
|
END IF;
|
|
8811
8923
|
RETURN NEW;
|
|
@@ -13410,7 +13522,7 @@ var init_sleep = __esm({
|
|
|
13410
13522
|
"node_modules/@smithy/core/dist-es/submodules/client/util-waiter/utils/sleep.js"() {
|
|
13411
13523
|
"use strict";
|
|
13412
13524
|
sleep = (seconds) => {
|
|
13413
|
-
return new Promise((
|
|
13525
|
+
return new Promise((resolve17) => setTimeout(resolve17, seconds * 1e3));
|
|
13414
13526
|
};
|
|
13415
13527
|
}
|
|
13416
13528
|
});
|
|
@@ -13579,8 +13691,8 @@ var init_createWaiter = __esm({
|
|
|
13579
13691
|
init_waiter2();
|
|
13580
13692
|
abortTimeout = (abortSignal) => {
|
|
13581
13693
|
let onAbort;
|
|
13582
|
-
const promise = new Promise((
|
|
13583
|
-
onAbort = () =>
|
|
13694
|
+
const promise = new Promise((resolve17) => {
|
|
13695
|
+
onAbort = () => resolve17({ state: WaiterState.ABORTED });
|
|
13584
13696
|
if (typeof abortSignal.addEventListener === "function") {
|
|
13585
13697
|
abortSignal.addEventListener("abort", onAbort);
|
|
13586
13698
|
} else {
|
|
@@ -16462,7 +16574,7 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
16462
16574
|
};
|
|
16463
16575
|
imdsHttpGet = async ({ hostname, path: path2 }) => {
|
|
16464
16576
|
const { request } = await import("http");
|
|
16465
|
-
return new Promise((
|
|
16577
|
+
return new Promise((resolve17, reject) => {
|
|
16466
16578
|
const req = request({
|
|
16467
16579
|
method: "GET",
|
|
16468
16580
|
hostname: hostname.replace(/^\[(.+)]$/, "$1"),
|
|
@@ -16488,7 +16600,7 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
16488
16600
|
const chunks = [];
|
|
16489
16601
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
16490
16602
|
res.on("end", () => {
|
|
16491
|
-
|
|
16603
|
+
resolve17(Buffer.concat(chunks));
|
|
16492
16604
|
req.destroy();
|
|
16493
16605
|
});
|
|
16494
16606
|
});
|
|
@@ -18248,7 +18360,7 @@ async function collectStream(stream) {
|
|
|
18248
18360
|
return collected;
|
|
18249
18361
|
}
|
|
18250
18362
|
function readToBase64(blob) {
|
|
18251
|
-
return new Promise((
|
|
18363
|
+
return new Promise((resolve17, reject) => {
|
|
18252
18364
|
const reader = new FileReader();
|
|
18253
18365
|
reader.onloadend = () => {
|
|
18254
18366
|
if (reader.readyState !== 2) {
|
|
@@ -18257,7 +18369,7 @@ function readToBase64(blob) {
|
|
|
18257
18369
|
const result = reader.result ?? "";
|
|
18258
18370
|
const commaIndex = result.indexOf(",");
|
|
18259
18371
|
const dataOffset = commaIndex > -1 ? commaIndex + 1 : result.length;
|
|
18260
|
-
|
|
18372
|
+
resolve17(result.substring(dataOffset));
|
|
18261
18373
|
};
|
|
18262
18374
|
reader.onabort = () => reject(new Error("Read aborted"));
|
|
18263
18375
|
reader.onerror = () => reject(reader.error);
|
|
@@ -18385,7 +18497,7 @@ var init_stream_collector = __esm({
|
|
|
18385
18497
|
if (isReadableStreamInstance(stream)) {
|
|
18386
18498
|
return collectReadableStream(stream);
|
|
18387
18499
|
}
|
|
18388
|
-
return new Promise((
|
|
18500
|
+
return new Promise((resolve17, reject) => {
|
|
18389
18501
|
const collector = new Collector();
|
|
18390
18502
|
stream.pipe(collector);
|
|
18391
18503
|
stream.on("error", (err) => {
|
|
@@ -18395,7 +18507,7 @@ var init_stream_collector = __esm({
|
|
|
18395
18507
|
collector.on("error", reject);
|
|
18396
18508
|
collector.on("finish", function() {
|
|
18397
18509
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
18398
|
-
|
|
18510
|
+
resolve17(bytes);
|
|
18399
18511
|
});
|
|
18400
18512
|
});
|
|
18401
18513
|
};
|
|
@@ -18542,11 +18654,11 @@ var init_SerdeContext = __esm({
|
|
|
18542
18654
|
// node_modules/tslib/tslib.es6.mjs
|
|
18543
18655
|
function __awaiter(thisArg, _arguments, P2, generator) {
|
|
18544
18656
|
function adopt(value) {
|
|
18545
|
-
return value instanceof P2 ? value : new P2(function(
|
|
18546
|
-
|
|
18657
|
+
return value instanceof P2 ? value : new P2(function(resolve17) {
|
|
18658
|
+
resolve17(value);
|
|
18547
18659
|
});
|
|
18548
18660
|
}
|
|
18549
|
-
return new (P2 || (P2 = Promise))(function(
|
|
18661
|
+
return new (P2 || (P2 = Promise))(function(resolve17, reject) {
|
|
18550
18662
|
function fulfilled(value) {
|
|
18551
18663
|
try {
|
|
18552
18664
|
step(generator.next(value));
|
|
@@ -18562,7 +18674,7 @@ function __awaiter(thisArg, _arguments, P2, generator) {
|
|
|
18562
18674
|
}
|
|
18563
18675
|
}
|
|
18564
18676
|
function step(result) {
|
|
18565
|
-
result.done ?
|
|
18677
|
+
result.done ? resolve17(result.value) : adopt(result.value).then(fulfilled, rejected);
|
|
18566
18678
|
}
|
|
18567
18679
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
18568
18680
|
});
|
|
@@ -19759,7 +19871,7 @@ async function* readableToIterable(readStream) {
|
|
|
19759
19871
|
streamEnded = true;
|
|
19760
19872
|
});
|
|
19761
19873
|
while (!generationEnded) {
|
|
19762
|
-
const value = await new Promise((
|
|
19874
|
+
const value = await new Promise((resolve17) => setTimeout(() => resolve17(records.shift()), 0));
|
|
19763
19875
|
if (value) {
|
|
19764
19876
|
yield value;
|
|
19765
19877
|
}
|
|
@@ -21304,7 +21416,7 @@ var init_retryMiddleware = __esm({
|
|
|
21304
21416
|
init_constants5();
|
|
21305
21417
|
init_parseRetryAfterHeader();
|
|
21306
21418
|
init_util2();
|
|
21307
|
-
cooldown = (ms) => new Promise((
|
|
21419
|
+
cooldown = (ms) => new Promise((resolve17) => setTimeout(resolve17, ms));
|
|
21308
21420
|
isRetryStrategyV2 = (retryStrategy) => typeof retryStrategy.acquireInitialRetryToken !== "undefined" && typeof retryStrategy.refreshRetryTokenForRetry !== "undefined" && typeof retryStrategy.recordSuccess !== "undefined";
|
|
21309
21421
|
getRetryErrorInfo = (error, logger2) => {
|
|
21310
21422
|
const errorInfo = {
|
|
@@ -21403,7 +21515,7 @@ var init_DefaultRateLimiter = __esm({
|
|
|
21403
21515
|
this.refillTokenBucket();
|
|
21404
21516
|
while (amount > this.availableTokens) {
|
|
21405
21517
|
const delay = (amount - this.availableTokens) / this.fillRate * 1e3;
|
|
21406
|
-
await new Promise((
|
|
21518
|
+
await new Promise((resolve17) => _DefaultRateLimiter.setTimeoutFn(resolve17, delay));
|
|
21407
21519
|
this.refillTokenBucket();
|
|
21408
21520
|
}
|
|
21409
21521
|
this.availableTokens = this.availableTokens - amount;
|
|
@@ -25073,8 +25185,8 @@ var init_SignatureV4 = __esm({
|
|
|
25073
25185
|
priorSignature: signableMessage.priorSignature,
|
|
25074
25186
|
eventStreamCredentials
|
|
25075
25187
|
});
|
|
25076
|
-
return promise.then((
|
|
25077
|
-
return { message: signableMessage.message, signature };
|
|
25188
|
+
return promise.then((signature2) => {
|
|
25189
|
+
return { message: signableMessage.message, signature: signature2 };
|
|
25078
25190
|
});
|
|
25079
25191
|
}
|
|
25080
25192
|
async signString(stringToSign, { signingDate = /* @__PURE__ */ new Date(), signingRegion, signingService, eventStreamCredentials } = {}) {
|
|
@@ -25102,8 +25214,8 @@ var init_SignatureV4 = __esm({
|
|
|
25102
25214
|
request.headers[SHA256_HEADER] = payloadHash;
|
|
25103
25215
|
}
|
|
25104
25216
|
const canonicalHeaders = getCanonicalHeaders(request, unsignableHeaders, signableHeaders);
|
|
25105
|
-
const
|
|
25106
|
-
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${
|
|
25217
|
+
const signature2 = await this.getSignature(longDate, scope, this.getSigningKey(credentials, region, shortDate, signingService), this.createCanonicalRequest(request, canonicalHeaders, payloadHash));
|
|
25218
|
+
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${signature2}`;
|
|
25107
25219
|
return request;
|
|
25108
25220
|
}
|
|
25109
25221
|
async getSignature(longDate, credentialScope, keyPromise, canonicalRequest) {
|
|
@@ -40691,7 +40803,7 @@ var init_node_http = __esm({
|
|
|
40691
40803
|
|
|
40692
40804
|
// node_modules/@smithy/credential-provider-imds/dist-es/remoteProvider/httpRequest.js
|
|
40693
40805
|
function httpRequest(options) {
|
|
40694
|
-
return new Promise((
|
|
40806
|
+
return new Promise((resolve17, reject) => {
|
|
40695
40807
|
const req = node_http.request({
|
|
40696
40808
|
method: "GET",
|
|
40697
40809
|
...options,
|
|
@@ -40716,7 +40828,7 @@ function httpRequest(options) {
|
|
|
40716
40828
|
chunks.push(chunk);
|
|
40717
40829
|
});
|
|
40718
40830
|
res.on("end", () => {
|
|
40719
|
-
|
|
40831
|
+
resolve17(Buffer.concat(chunks));
|
|
40720
40832
|
req.destroy();
|
|
40721
40833
|
});
|
|
40722
40834
|
});
|
|
@@ -41361,21 +41473,21 @@ async function writeRequestBody(httpRequest2, request, maxContinueTimeoutMs = MI
|
|
|
41361
41473
|
let sendBody = true;
|
|
41362
41474
|
if (!externalAgent && expect === "100-continue") {
|
|
41363
41475
|
sendBody = await Promise.race([
|
|
41364
|
-
new Promise((
|
|
41365
|
-
timeoutId = Number(timing.setTimeout(() =>
|
|
41476
|
+
new Promise((resolve17) => {
|
|
41477
|
+
timeoutId = Number(timing.setTimeout(() => resolve17(true), Math.max(MIN_WAIT_TIME, maxContinueTimeoutMs)));
|
|
41366
41478
|
}),
|
|
41367
|
-
new Promise((
|
|
41479
|
+
new Promise((resolve17) => {
|
|
41368
41480
|
httpRequest2.on("continue", () => {
|
|
41369
41481
|
timing.clearTimeout(timeoutId);
|
|
41370
|
-
|
|
41482
|
+
resolve17(true);
|
|
41371
41483
|
});
|
|
41372
41484
|
httpRequest2.on("response", () => {
|
|
41373
41485
|
timing.clearTimeout(timeoutId);
|
|
41374
|
-
|
|
41486
|
+
resolve17(false);
|
|
41375
41487
|
});
|
|
41376
41488
|
httpRequest2.on("error", () => {
|
|
41377
41489
|
timing.clearTimeout(timeoutId);
|
|
41378
|
-
|
|
41490
|
+
resolve17(false);
|
|
41379
41491
|
});
|
|
41380
41492
|
})
|
|
41381
41493
|
]);
|
|
@@ -41473,13 +41585,13 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41473
41585
|
return socketWarningTimestamp;
|
|
41474
41586
|
}
|
|
41475
41587
|
constructor(options) {
|
|
41476
|
-
this.configProvider = new Promise((
|
|
41588
|
+
this.configProvider = new Promise((resolve17, reject) => {
|
|
41477
41589
|
if (typeof options === "function") {
|
|
41478
41590
|
options().then((_options) => {
|
|
41479
|
-
|
|
41591
|
+
resolve17(this.resolveDefaultConfig(_options));
|
|
41480
41592
|
}).catch(reject);
|
|
41481
41593
|
} else {
|
|
41482
|
-
|
|
41594
|
+
resolve17(this.resolveDefaultConfig(options));
|
|
41483
41595
|
}
|
|
41484
41596
|
});
|
|
41485
41597
|
}
|
|
@@ -41510,7 +41622,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41510
41622
|
timing.clearTimeout(socketTimeoutId);
|
|
41511
41623
|
timing.clearTimeout(keepAliveTimeoutId);
|
|
41512
41624
|
};
|
|
41513
|
-
const
|
|
41625
|
+
const resolve17 = async (arg) => {
|
|
41514
41626
|
await writeRequestBodyPromise;
|
|
41515
41627
|
clearTimeouts();
|
|
41516
41628
|
_resolve(arg);
|
|
@@ -41574,7 +41686,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41574
41686
|
headers: getTransformedHeaders(res.headers),
|
|
41575
41687
|
body: res
|
|
41576
41688
|
});
|
|
41577
|
-
|
|
41689
|
+
resolve17({ response: httpResponse });
|
|
41578
41690
|
});
|
|
41579
41691
|
req.on("error", (err) => {
|
|
41580
41692
|
if (NODEJS_TIMEOUT_ERROR_CODES2.includes(err.code)) {
|
|
@@ -41726,7 +41838,7 @@ var init_stream_collector2 = __esm({
|
|
|
41726
41838
|
if (isReadableStreamInstance2(stream)) {
|
|
41727
41839
|
return collectReadableStream2(stream);
|
|
41728
41840
|
}
|
|
41729
|
-
return new Promise((
|
|
41841
|
+
return new Promise((resolve17, reject) => {
|
|
41730
41842
|
const collector = new Collector2();
|
|
41731
41843
|
stream.pipe(collector);
|
|
41732
41844
|
stream.on("error", (err) => {
|
|
@@ -41736,7 +41848,7 @@ var init_stream_collector2 = __esm({
|
|
|
41736
41848
|
collector.on("error", reject);
|
|
41737
41849
|
collector.on("finish", function() {
|
|
41738
41850
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
41739
|
-
|
|
41851
|
+
resolve17(bytes);
|
|
41740
41852
|
});
|
|
41741
41853
|
});
|
|
41742
41854
|
};
|
|
@@ -41858,7 +41970,7 @@ var init_retry_wrapper = __esm({
|
|
|
41858
41970
|
try {
|
|
41859
41971
|
return await toRetry();
|
|
41860
41972
|
} catch (e6) {
|
|
41861
|
-
await new Promise((
|
|
41973
|
+
await new Promise((resolve17) => setTimeout(resolve17, delayMs));
|
|
41862
41974
|
}
|
|
41863
41975
|
}
|
|
41864
41976
|
return await toRetry();
|
|
@@ -47093,14 +47205,14 @@ var init_readableStreamHasher = __esm({
|
|
|
47093
47205
|
const hash = new hashCtor();
|
|
47094
47206
|
const hashCalculator = new HashCalculator(hash);
|
|
47095
47207
|
readableStream.pipe(hashCalculator);
|
|
47096
|
-
return new Promise((
|
|
47208
|
+
return new Promise((resolve17, reject) => {
|
|
47097
47209
|
readableStream.on("error", (err) => {
|
|
47098
47210
|
hashCalculator.end();
|
|
47099
47211
|
reject(err);
|
|
47100
47212
|
});
|
|
47101
47213
|
hashCalculator.on("error", reject);
|
|
47102
47214
|
hashCalculator.on("finish", () => {
|
|
47103
|
-
hash.digest().then(
|
|
47215
|
+
hash.digest().then(resolve17).catch(reject);
|
|
47104
47216
|
});
|
|
47105
47217
|
});
|
|
47106
47218
|
};
|
|
@@ -52618,7 +52730,7 @@ function parsePageParam(raw, kind) {
|
|
|
52618
52730
|
}
|
|
52619
52731
|
function readJson(req, opts = {}) {
|
|
52620
52732
|
const maxBytes = opts.maxBytes ?? DEFAULT_BODY_MAX_BYTES;
|
|
52621
|
-
return new Promise((
|
|
52733
|
+
return new Promise((resolve17, reject) => {
|
|
52622
52734
|
let raw = "";
|
|
52623
52735
|
let overflowed = false;
|
|
52624
52736
|
req.setEncoding("utf8");
|
|
@@ -52634,7 +52746,7 @@ function readJson(req, opts = {}) {
|
|
|
52634
52746
|
req.on("end", () => {
|
|
52635
52747
|
if (overflowed) return;
|
|
52636
52748
|
try {
|
|
52637
|
-
|
|
52749
|
+
resolve17(raw ? JSON.parse(raw) : {});
|
|
52638
52750
|
} catch {
|
|
52639
52751
|
reject(new Error("Invalid JSON body"));
|
|
52640
52752
|
}
|
|
@@ -52654,11 +52766,12 @@ ${err.stack ?? ""}`);
|
|
|
52654
52766
|
sendJson(res, { error: err.message }, status);
|
|
52655
52767
|
}
|
|
52656
52768
|
}
|
|
52657
|
-
var DEFAULT_BODY_MAX_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
52769
|
+
var DEFAULT_BODY_MAX_BYTES, MAX_INGEST_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
52658
52770
|
var init_http2 = __esm({
|
|
52659
52771
|
"src/gui/http.ts"() {
|
|
52660
52772
|
"use strict";
|
|
52661
52773
|
DEFAULT_BODY_MAX_BYTES = 1e6;
|
|
52774
|
+
MAX_INGEST_BYTES = 5e7;
|
|
52662
52775
|
MAX_ROWS_PAGE = 1e3;
|
|
52663
52776
|
DEFAULT_ROWS_PAGE = 500;
|
|
52664
52777
|
BodyTooLargeError = class extends Error {
|
|
@@ -53651,7 +53764,7 @@ var init_oauth = __esm({
|
|
|
53651
53764
|
|
|
53652
53765
|
// src/gui/assistant-routes.ts
|
|
53653
53766
|
function readBuffer(req, maxBytes = 25e6) {
|
|
53654
|
-
return new Promise((
|
|
53767
|
+
return new Promise((resolve17, reject) => {
|
|
53655
53768
|
const chunks = [];
|
|
53656
53769
|
let size = 0;
|
|
53657
53770
|
req.on("data", (c6) => {
|
|
@@ -53660,7 +53773,7 @@ function readBuffer(req, maxBytes = 25e6) {
|
|
|
53660
53773
|
else chunks.push(c6);
|
|
53661
53774
|
});
|
|
53662
53775
|
req.on("end", () => {
|
|
53663
|
-
|
|
53776
|
+
resolve17(Buffer.concat(chunks));
|
|
53664
53777
|
});
|
|
53665
53778
|
req.on("error", reject);
|
|
53666
53779
|
});
|
|
@@ -54961,7 +55074,7 @@ async function takeHostSlot(host, minIntervalMs = urlIngestConfig().hostMinInter
|
|
|
54961
55074
|
const earliest = Math.max(now2, hostNextAllowed.get(key) ?? 0);
|
|
54962
55075
|
hostNextAllowed.set(key, earliest + minIntervalMs);
|
|
54963
55076
|
const wait = earliest - now2;
|
|
54964
|
-
if (wait > 0) await new Promise((
|
|
55077
|
+
if (wait > 0) await new Promise((resolve17) => setTimeout(resolve17, wait));
|
|
54965
55078
|
}
|
|
54966
55079
|
var Semaphore, FetchBudget, sharedGate, hostNextAllowed;
|
|
54967
55080
|
var init_fetch_policy = __esm({
|
|
@@ -54977,7 +55090,7 @@ var init_fetch_policy = __esm({
|
|
|
54977
55090
|
if (this.permits > 0) {
|
|
54978
55091
|
this.permits -= 1;
|
|
54979
55092
|
} else {
|
|
54980
|
-
await new Promise((
|
|
55093
|
+
await new Promise((resolve17) => this.waiters.push(resolve17));
|
|
54981
55094
|
}
|
|
54982
55095
|
let released = false;
|
|
54983
55096
|
return () => {
|
|
@@ -55214,8 +55327,8 @@ function fileContentGroups(rows, fuzzy, threshold) {
|
|
|
55214
55327
|
const t8 = get2(r6, "extracted_text");
|
|
55215
55328
|
return typeof t8 === "string" && t8.trim().length > 0;
|
|
55216
55329
|
}).map((r6) => {
|
|
55217
|
-
const
|
|
55218
|
-
const key = fuzzy ? "txt:" +
|
|
55330
|
+
const norm3 = normalizeText(get2(r6, "extracted_text"));
|
|
55331
|
+
const key = fuzzy ? "txt:" + norm3.slice(0, 2e3) : "txt:" + createHash11("sha256").update(norm3).digest("hex");
|
|
55219
55332
|
return { id: String(get2(r6, "id")), key, createdAt: cellStrOrNull(get2(r6, "created_at")) };
|
|
55220
55333
|
});
|
|
55221
55334
|
const txtGroups = findDuplicateGroups(txtItems, {
|
|
@@ -58123,8 +58236,8 @@ function isRetryableDbError(err) {
|
|
|
58123
58236
|
return msg.includes("database is locked") || msg.includes("connection terminated") || msg.includes("connection reset") || msg.includes("server closed the connection");
|
|
58124
58237
|
}
|
|
58125
58238
|
var retryDepth = new AsyncLocalStorage();
|
|
58126
|
-
var defaultSleep = (ms) => new Promise((
|
|
58127
|
-
setTimeout(
|
|
58239
|
+
var defaultSleep = (ms) => new Promise((resolve17) => {
|
|
58240
|
+
setTimeout(resolve17, ms);
|
|
58128
58241
|
});
|
|
58129
58242
|
async function withRetry(fn, opts = {}) {
|
|
58130
58243
|
if (retryDepth.getStore()) return fn();
|
|
@@ -58323,13 +58436,13 @@ import { createHash as createHash4 } from "crypto";
|
|
|
58323
58436
|
import { createReadStream, existsSync as existsSync12, mkdirSync as mkdirSync6, statSync as statSync5, copyFileSync as copyFileSync3 } from "fs";
|
|
58324
58437
|
import { basename as basename3, join as join12 } from "path";
|
|
58325
58438
|
async function hashFile(srcPath) {
|
|
58326
|
-
return new Promise((
|
|
58439
|
+
return new Promise((resolve17, reject) => {
|
|
58327
58440
|
const hash = createHash4("sha256");
|
|
58328
58441
|
const stream = createReadStream(srcPath);
|
|
58329
58442
|
stream.on("data", (chunk) => hash.update(chunk));
|
|
58330
58443
|
stream.on("error", reject);
|
|
58331
58444
|
stream.on("end", () => {
|
|
58332
|
-
|
|
58445
|
+
resolve17(hash.digest("hex"));
|
|
58333
58446
|
});
|
|
58334
58447
|
});
|
|
58335
58448
|
}
|
|
@@ -59528,7 +59641,7 @@ init_http2();
|
|
|
59528
59641
|
import { createServer } from "http";
|
|
59529
59642
|
import { spawn as spawn2 } from "child_process";
|
|
59530
59643
|
import { WebSocketServer, WebSocket } from "ws";
|
|
59531
|
-
import { dirname as dirname16, resolve as
|
|
59644
|
+
import { dirname as dirname16, resolve as resolve15 } from "path";
|
|
59532
59645
|
|
|
59533
59646
|
// src/gui/active-db.ts
|
|
59534
59647
|
init_adapter();
|
|
@@ -59536,9 +59649,17 @@ init_members();
|
|
|
59536
59649
|
init_native_entities();
|
|
59537
59650
|
async function changeVisibleToActiveRole(db, payload) {
|
|
59538
59651
|
if (db.getDialect() !== "postgres") return true;
|
|
59539
|
-
if (payload.op === "delete" || payload.op === "DELETE") return true;
|
|
59540
59652
|
if (!payload.table_name || !payload.pk) return false;
|
|
59541
59653
|
try {
|
|
59654
|
+
if (isDeleteOp(payload.op)) {
|
|
59655
|
+
if (payload.del_owner_role == null) return false;
|
|
59656
|
+
const row2 = await getAsyncOrSync(
|
|
59657
|
+
db.adapter,
|
|
59658
|
+
`SELECT lattice_delete_visible(?, ?, ?::text[]) AS v`,
|
|
59659
|
+
[payload.del_owner_role, payload.del_visibility ?? null, payload.del_grantees ?? []]
|
|
59660
|
+
);
|
|
59661
|
+
return row2?.v === true || row2?.v === "t" || row2?.v === 1;
|
|
59662
|
+
}
|
|
59542
59663
|
const row = await getAsyncOrSync(db.adapter, `SELECT lattice_row_visible(?, ?) AS v`, [
|
|
59543
59664
|
payload.table_name,
|
|
59544
59665
|
payload.pk
|
|
@@ -60029,9 +60150,9 @@ var RealtimeBroker = class {
|
|
|
60029
60150
|
() => "ended"
|
|
60030
60151
|
// a graceful-close error is still "closed enough"
|
|
60031
60152
|
);
|
|
60032
|
-
const timedOut = new Promise((
|
|
60153
|
+
const timedOut = new Promise((resolve17) => {
|
|
60033
60154
|
timer = setTimeout(() => {
|
|
60034
|
-
|
|
60155
|
+
resolve17("timeout");
|
|
60035
60156
|
}, this.stopEndTimeoutMs);
|
|
60036
60157
|
timer.unref?.();
|
|
60037
60158
|
});
|
|
@@ -60076,7 +60197,10 @@ function parsePayload(raw) {
|
|
|
60076
60197
|
pk: typeof obj2.pk === "string" ? obj2.pk : null,
|
|
60077
60198
|
op: obj2.op,
|
|
60078
60199
|
owner_role: typeof obj2.owner_role === "string" ? obj2.owner_role : null,
|
|
60079
|
-
created_at: typeof obj2.created_at === "string" ? obj2.created_at : ""
|
|
60200
|
+
created_at: typeof obj2.created_at === "string" ? obj2.created_at : "",
|
|
60201
|
+
del_owner_role: typeof obj2.del_owner_role === "string" ? obj2.del_owner_role : null,
|
|
60202
|
+
del_visibility: typeof obj2.del_visibility === "string" ? obj2.del_visibility : null,
|
|
60203
|
+
del_grantees: Array.isArray(obj2.del_grantees) ? obj2.del_grantees.filter((g6) => typeof g6 === "string") : null
|
|
60080
60204
|
};
|
|
60081
60205
|
} catch {
|
|
60082
60206
|
return null;
|
|
@@ -61057,9 +61181,9 @@ function startBackgroundRender(active) {
|
|
|
61057
61181
|
}
|
|
61058
61182
|
function settleWithin(p3, ms) {
|
|
61059
61183
|
let timer;
|
|
61060
|
-
const timeout = new Promise((
|
|
61184
|
+
const timeout = new Promise((resolve17) => {
|
|
61061
61185
|
timer = setTimeout(() => {
|
|
61062
|
-
|
|
61186
|
+
resolve17("timeout");
|
|
61063
61187
|
}, ms);
|
|
61064
61188
|
timer.unref?.();
|
|
61065
61189
|
});
|
|
@@ -61114,9 +61238,9 @@ var SWITCH_OPEN_TIMEOUT_MS = 2e4;
|
|
|
61114
61238
|
async function openWithinTimeout(open, timeoutMs = SWITCH_OPEN_TIMEOUT_MS, dispose = disposeActive) {
|
|
61115
61239
|
const opening = open();
|
|
61116
61240
|
let timer;
|
|
61117
|
-
const timedOut = new Promise((
|
|
61241
|
+
const timedOut = new Promise((resolve17) => {
|
|
61118
61242
|
timer = setTimeout(() => {
|
|
61119
|
-
|
|
61243
|
+
resolve17("timeout");
|
|
61120
61244
|
}, timeoutMs);
|
|
61121
61245
|
timer.unref?.();
|
|
61122
61246
|
});
|
|
@@ -62787,6 +62911,65 @@ var chatCss = ` /* \u2500\u2500 Chat bubbles + tool pills \u2500\u2500\u2500\
|
|
|
62787
62911
|
}
|
|
62788
62912
|
`;
|
|
62789
62913
|
|
|
62914
|
+
// src/gui/app/styles/inline-import.ts
|
|
62915
|
+
var inlineImportCss = `
|
|
62916
|
+
/* \u2500\u2500 Inline import confirm card (assistant rail) \u2500\u2500 */
|
|
62917
|
+
.cd-sub { margin: 10px 0 6px; font-size: 12px; color: var(--text-muted, #9aa3ad); }
|
|
62918
|
+
.cd-row { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; margin-top: 8px; }
|
|
62919
|
+
.cd-path {
|
|
62920
|
+
flex: 1 1 220px; min-width: 0; box-sizing: border-box; height: 34px; padding: 0 10px;
|
|
62921
|
+
border-radius: 6px; border: 1px solid #2a2f36;
|
|
62922
|
+
background: var(--panel, #0e1116); color: var(--text, #e6e8eb); font-size: 13px;
|
|
62923
|
+
}
|
|
62924
|
+
.cd-status { margin-top: 12px; font-size: 13px; line-height: 1.5; }
|
|
62925
|
+
.cd-status.ok { color: #bef264; }
|
|
62926
|
+
.cd-status.err { color: #f87171; }
|
|
62927
|
+
.cd-status a { color: var(--accent, #bef264); }
|
|
62928
|
+
.cd-btn {
|
|
62929
|
+
height: 34px; padding: 0 14px; border-radius: 6px; border: 1px solid #2a2f36;
|
|
62930
|
+
background: transparent; color: var(--text, #e6e8eb); font-size: 13px;
|
|
62931
|
+
font-weight: 600; cursor: pointer;
|
|
62932
|
+
}
|
|
62933
|
+
.cd-btn:hover { background: rgba(255, 255, 255, 0.06); }
|
|
62934
|
+
.cd-btn.cd-primary { background: #bef264; color: #0b0d10; border-color: #bef264; }
|
|
62935
|
+
.cd-btn.cd-primary:hover { filter: brightness(1.06); }
|
|
62936
|
+
.cd-import-list { margin: 10px 0 0; padding-left: 18px; font-size: 13px; line-height: 1.6; }
|
|
62937
|
+
.cd-import-list li { margin: 2px 0; }
|
|
62938
|
+
.imp-sub { margin: 16px 0 6px; font-size: 13px; color: var(--text, #e6e8eb); }
|
|
62939
|
+
.imp-modes { display: flex; flex-direction: column; gap: 8px; margin: 0 0 6px; }
|
|
62940
|
+
.imp-modes label {
|
|
62941
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
62942
|
+
padding: 8px 10px; border: 1px solid #2a2f36; border-radius: 6px; cursor: pointer;
|
|
62943
|
+
}
|
|
62944
|
+
.imp-modes label:hover { background: rgba(255, 255, 255, 0.04); }
|
|
62945
|
+
.imp-modes input { margin-top: 2px; }
|
|
62946
|
+
.imp-modes b { color: var(--text, #e6e8eb); }
|
|
62947
|
+
.imp-percol {
|
|
62948
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
62949
|
+
margin: 8px 0 0; cursor: pointer; color: var(--text-dim, #aeb6c2);
|
|
62950
|
+
}
|
|
62951
|
+
.imp-percol input { margin-top: 2px; }
|
|
62952
|
+
.imp-match { border-left: 3px solid var(--accent, #7dd3fc); font-weight: 500; }
|
|
62953
|
+
.feed-item.import-confirm .imp-confirm-body { margin-top: 4px; }
|
|
62954
|
+
|
|
62955
|
+
/* \u2500\u2500 Live import progress in the card's log \u2500\u2500 */
|
|
62956
|
+
.feed-item.import-confirm .imp-card-log,
|
|
62957
|
+
.feed-item.import-live .imp-card-log {
|
|
62958
|
+
margin-top: 4px;
|
|
62959
|
+
font: 12px/1.6 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
|
62960
|
+
max-height: 200px; overflow-y: auto; color: var(--text-muted, #9aa3ad);
|
|
62961
|
+
}
|
|
62962
|
+
.imp-card-line { white-space: pre-wrap; word-break: break-word; }
|
|
62963
|
+
.imp-card-line.imp-done { color: var(--accent, #bef264); }
|
|
62964
|
+
.imp-card-line.imp-err { color: #f87171; }
|
|
62965
|
+
.imp-card-line.imp-spin::after {
|
|
62966
|
+
content: ''; display: inline-block; width: 10px; height: 10px; margin-left: 7px;
|
|
62967
|
+
border: 2px solid currentColor; border-right-color: transparent; border-radius: 50%;
|
|
62968
|
+
vertical-align: -1px; animation: imp-spin-kf 0.7s linear infinite;
|
|
62969
|
+
}
|
|
62970
|
+
@keyframes imp-spin-kf { to { transform: rotate(360deg); } }
|
|
62971
|
+
`;
|
|
62972
|
+
|
|
62790
62973
|
// src/gui/app/styles/index.ts
|
|
62791
62974
|
var css = [
|
|
62792
62975
|
tokensCss,
|
|
@@ -62808,7 +62991,8 @@ var css = [
|
|
|
62808
62991
|
fsWorkspaceCss,
|
|
62809
62992
|
settingsDrawerCss,
|
|
62810
62993
|
assistantRailCss,
|
|
62811
|
-
chatCss
|
|
62994
|
+
chatCss,
|
|
62995
|
+
inlineImportCss
|
|
62812
62996
|
].join("");
|
|
62813
62997
|
|
|
62814
62998
|
// src/gui/app/modules/display-config.ts
|
|
@@ -70379,6 +70563,11 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
70379
70563
|
// survivor if it was a duplicate). Multi-file drops do not navigate.
|
|
70380
70564
|
if (files.length === 1) {
|
|
70381
70565
|
uploadFile(files[0]).then(function (j) {
|
|
70566
|
+
// A structured source the server flagged as confirmable comes back with
|
|
70567
|
+
// an autoImport proposal \u2014 render the inline confirm card instead of
|
|
70568
|
+
// navigating to the file record. A silent import (autoImport.imported,
|
|
70569
|
+
// no reason) or a plain file keeps the open-the-record behavior.
|
|
70570
|
+
if (j && j.autoImport && j.autoImport.reason) { renderInlineImportCard(j.autoImport); return; }
|
|
70382
70571
|
if (j && (j.duplicateOf || j.id)) openSearchHit('files', j.duplicateOf || j.id);
|
|
70383
70572
|
});
|
|
70384
70573
|
return;
|
|
@@ -70388,7 +70577,15 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
70388
70577
|
var bar = ingestProgress(files.length);
|
|
70389
70578
|
var thunks = [];
|
|
70390
70579
|
for (var i = 0; i < files.length; i++) {
|
|
70391
|
-
(function (f) {
|
|
70580
|
+
(function (f) {
|
|
70581
|
+
thunks.push(function () {
|
|
70582
|
+
return uploadFile(f).then(function (j) {
|
|
70583
|
+
// A structured source within a batch still gets its own inline
|
|
70584
|
+
// confirm card (the batch as a whole does not navigate).
|
|
70585
|
+
if (j && j.autoImport && j.autoImport.reason) renderInlineImportCard(j.autoImport);
|
|
70586
|
+
});
|
|
70587
|
+
});
|
|
70588
|
+
})(files[i]);
|
|
70392
70589
|
}
|
|
70393
70590
|
runIngestBatch(thunks, INGEST_MAX_CONCURRENCY, bar.update).then(bar.done);
|
|
70394
70591
|
}
|
|
@@ -70544,6 +70741,237 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
70544
70741
|
})();
|
|
70545
70742
|
`;
|
|
70546
70743
|
|
|
70744
|
+
// src/gui/app/modules/inline-import.ts
|
|
70745
|
+
var inlineImportJs = `
|
|
70746
|
+
// \u2500\u2500 Inline structured-source import (confirm card in the assistant rail) \u2500\u2500
|
|
70747
|
+
function iiRailFeed() { return document.getElementById('rail-feed'); }
|
|
70748
|
+
function iiRailEmptyGone() {
|
|
70749
|
+
var e = document.getElementById('rail-empty');
|
|
70750
|
+
if (e) e.parentNode && e.parentNode.removeChild(e);
|
|
70751
|
+
}
|
|
70752
|
+
|
|
70753
|
+
// Read a newline-delimited-JSON response body, invoking onEvent(obj) per line.
|
|
70754
|
+
// Self-contained on purpose \u2014 this segment must not depend on any other.
|
|
70755
|
+
function iiStreamNdjson(url, payload, onEvent) {
|
|
70756
|
+
fetch(url, {
|
|
70757
|
+
method: 'POST',
|
|
70758
|
+
headers: { 'content-type': 'application/json' },
|
|
70759
|
+
body: JSON.stringify(payload),
|
|
70760
|
+
}).then(function (res) {
|
|
70761
|
+
if (!res.body || !res.body.getReader) {
|
|
70762
|
+
return res.text().then(function (t) {
|
|
70763
|
+
t.split('\\n').forEach(function (line) {
|
|
70764
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
70765
|
+
});
|
|
70766
|
+
});
|
|
70767
|
+
}
|
|
70768
|
+
var reader = res.body.getReader();
|
|
70769
|
+
var dec = new TextDecoder();
|
|
70770
|
+
var buf = '';
|
|
70771
|
+
function pump() {
|
|
70772
|
+
return reader.read().then(function (chunk) {
|
|
70773
|
+
if (chunk.done) {
|
|
70774
|
+
if (buf.trim()) { try { onEvent(JSON.parse(buf)); } catch (e) { /* skip */ } }
|
|
70775
|
+
return;
|
|
70776
|
+
}
|
|
70777
|
+
buf += dec.decode(chunk.value, { stream: true });
|
|
70778
|
+
var idx;
|
|
70779
|
+
while ((idx = buf.indexOf('\\n')) >= 0) {
|
|
70780
|
+
var line = buf.slice(0, idx);
|
|
70781
|
+
buf = buf.slice(idx + 1);
|
|
70782
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
70783
|
+
}
|
|
70784
|
+
return pump();
|
|
70785
|
+
});
|
|
70786
|
+
}
|
|
70787
|
+
return pump();
|
|
70788
|
+
}).catch(function (err) {
|
|
70789
|
+
onEvent({ phase: 'error', message: err && err.message ? err.message : 'Request failed' });
|
|
70790
|
+
});
|
|
70791
|
+
}
|
|
70792
|
+
|
|
70793
|
+
// Render the confirm card for a structured drop the server flagged as
|
|
70794
|
+
// needing confirmation. autoImport is the upload response's proposal:
|
|
70795
|
+
// { reason, fileId, plan:{entities,dimensions,linkages}, views, asOf,
|
|
70796
|
+
// asOfCandidates, asOfColumns, schemaMatch, matchedCount, totalEntities }.
|
|
70797
|
+
function renderInlineImportCard(autoImport) {
|
|
70798
|
+
if (!autoImport || !autoImport.fileId) return;
|
|
70799
|
+
var plan = autoImport.plan || {};
|
|
70800
|
+
var ents = plan.entities || [];
|
|
70801
|
+
var dims = plan.dimensions || [];
|
|
70802
|
+
var links = plan.linkages || [];
|
|
70803
|
+
var views = autoImport.views || [];
|
|
70804
|
+
var candidates = autoImport.asOfCandidates || [];
|
|
70805
|
+
var asOfColumns = autoImport.asOfColumns || [];
|
|
70806
|
+
var schemaMatch = autoImport.schemaMatch || {};
|
|
70807
|
+
var headerText = autoImport.reason === 'needs-confirm'
|
|
70808
|
+
? 'Add a dated snapshot'
|
|
70809
|
+
: 'Import as a new dataset';
|
|
70810
|
+
|
|
70811
|
+
iiRailEmptyGone();
|
|
70812
|
+
var feedEl = iiRailFeed();
|
|
70813
|
+
var card = document.createElement('div');
|
|
70814
|
+
card.className = 'feed-item import-confirm';
|
|
70815
|
+
var icon = document.createElement('div');
|
|
70816
|
+
icon.className = 'feed-icon';
|
|
70817
|
+
icon.textContent = '\u2913';
|
|
70818
|
+
var bodyEl = document.createElement('div');
|
|
70819
|
+
bodyEl.className = 'feed-body';
|
|
70820
|
+
var title = document.createElement('div');
|
|
70821
|
+
title.className = 'feed-summary';
|
|
70822
|
+
title.textContent = headerText;
|
|
70823
|
+
bodyEl.appendChild(title);
|
|
70824
|
+
|
|
70825
|
+
var parts = [];
|
|
70826
|
+
if (schemaMatch.isKnownDocument) {
|
|
70827
|
+
parts.push('<div class="cd-status ok imp-match">Recognized as a new period of an existing document — ' +
|
|
70828
|
+
schemaMatch.matchedCount + ' of ' + schemaMatch.totalEntities +
|
|
70829
|
+
' tables match what you already imported. It will be added as a dated snapshot.</div>');
|
|
70830
|
+
}
|
|
70831
|
+
parts.push('<div class="cd-status ok">Found ' + ents.length + ' entities, ' + dims.length +
|
|
70832
|
+
' dimensions, ' + links.length + ' links' +
|
|
70833
|
+
(views.length ? ', ' + views.length + ' reconstructed views (no duplicated rows)' : '') +
|
|
70834
|
+
'.</div><ul class="cd-import-list">');
|
|
70835
|
+
ents.forEach(function (e) {
|
|
70836
|
+
parts.push('<li><b>' + escapeHtml(e.name) + '</b> — ' + e.rowCount + ' rows, ' +
|
|
70837
|
+
(e.columns ? e.columns.length : 0) + ' cols · ' +
|
|
70838
|
+
(e.naturalKey ? 'key ' + escapeHtml(e.naturalKey) : 'keyless') + '</li>');
|
|
70839
|
+
});
|
|
70840
|
+
dims.forEach(function (d) {
|
|
70841
|
+
parts.push('<li><b>' + escapeHtml(d.name) + '</b> (dimension) — ' + d.distinctValues + ' values</li>');
|
|
70842
|
+
});
|
|
70843
|
+
views.forEach(function (v) {
|
|
70844
|
+
parts.push('<li><b>' + escapeHtml(v.name) + '</b> (view of ' + escapeHtml(v.master) + ' where ' +
|
|
70845
|
+
escapeHtml(v.filterColumn) + ' = ' + escapeHtml(String(v.filterValue)) + ') — ' +
|
|
70846
|
+
v.matchedRows + ' rows, not duplicated</li>');
|
|
70847
|
+
});
|
|
70848
|
+
parts.push('</ul>');
|
|
70849
|
+
|
|
70850
|
+
parts.push('<h4 class="imp-sub">As of date</h4>');
|
|
70851
|
+
var best = candidates[0];
|
|
70852
|
+
parts.push('<p class="cd-sub">' +
|
|
70853
|
+
(best ? 'Detected from ' + escapeHtml(best.evidence) + ' — edit if wrong.'
|
|
70854
|
+
: 'No date found in the file or its name — set the snapshot date, or leave blank to import undated.') +
|
|
70855
|
+
' A newer file is kept as a separate dated snapshot beside the prior one.</p>');
|
|
70856
|
+
parts.push('<div class="cd-row"><input class="cd-path" id="ii-asof" type="date" value="' + escapeHtml(autoImport.asOf || '') + '" aria-label="As of date" /></div>');
|
|
70857
|
+
if (candidates.length > 1) {
|
|
70858
|
+
parts.push('<div class="cd-sub">Other candidates: ' + candidates.slice(1, 5).map(function (c) {
|
|
70859
|
+
return '<a href="#" class="ii-asof-alt" data-date="' + escapeHtml(c.date) + '" title="' + escapeHtml(c.evidence) + '">' + escapeHtml(c.date) + '</a>';
|
|
70860
|
+
}).join(', ') + '</div>');
|
|
70861
|
+
}
|
|
70862
|
+
if (asOfColumns.length) {
|
|
70863
|
+
var colOpts = asOfColumns.slice(0, 6).map(function (c) {
|
|
70864
|
+
return '<option value="' + escapeHtml(c.column) + '" title="' + escapeHtml(c.evidence) + '">' +
|
|
70865
|
+
escapeHtml(c.column) + ' (' + escapeHtml(c.entity) + ', ' + c.distinctDates +
|
|
70866
|
+
' date' + (c.distinctDates === 1 ? '' : 's') + ')</option>';
|
|
70867
|
+
}).join('');
|
|
70868
|
+
parts.push('<label class="imp-percol"><input type="checkbox" id="ii-asof-percol"> ' +
|
|
70869
|
+
'<span>Date varies per row — use a date column instead (one file, many periods)</span></label>');
|
|
70870
|
+
parts.push('<div class="cd-row" id="ii-asof-col-row" style="display:none"><select class="cd-path" id="ii-asof-col">' + colOpts + '</select></div>');
|
|
70871
|
+
}
|
|
70872
|
+
|
|
70873
|
+
parts.push('<h4 class="imp-sub">What should Lattice bring in?</h4>');
|
|
70874
|
+
parts.push('<div class="imp-modes">' +
|
|
70875
|
+
'<label><input type="radio" name="ii-mode" value="both" checked> <span><b>Data model + contents</b> \u2014 the schema, the taxonomy, and all the rows.</span></label>' +
|
|
70876
|
+
'<label><input type="radio" name="ii-mode" value="schema"> <span><b>Data model / schema only</b> \u2014 tables, dimension values, and views. No rows.</span></label>' +
|
|
70877
|
+
'<label><input type="radio" name="ii-mode" value="contents"> <span><b>Contents only</b> \u2014 the rows and their links, into tables that already exist.</span></label>' +
|
|
70878
|
+
'</div>');
|
|
70879
|
+
parts.push('<div class="cd-row"><button class="cd-btn cd-primary" id="ii-apply" type="button">Import into Lattice</button></div>');
|
|
70880
|
+
parts.push('<div class="imp-card-log" id="ii-log"></div>');
|
|
70881
|
+
|
|
70882
|
+
var content = document.createElement('div');
|
|
70883
|
+
content.className = 'imp-confirm-body';
|
|
70884
|
+
content.innerHTML = parts.join('');
|
|
70885
|
+
bodyEl.appendChild(content);
|
|
70886
|
+
card.appendChild(icon);
|
|
70887
|
+
card.appendChild(bodyEl);
|
|
70888
|
+
if (feedEl) { feedEl.appendChild(card); feedEl.scrollTop = feedEl.scrollHeight; }
|
|
70889
|
+
|
|
70890
|
+
content.querySelectorAll('.ii-asof-alt').forEach(function (a) {
|
|
70891
|
+
a.addEventListener('click', function (e) {
|
|
70892
|
+
e.preventDefault();
|
|
70893
|
+
var input = document.getElementById('ii-asof');
|
|
70894
|
+
if (input) input.value = a.getAttribute('data-date') || '';
|
|
70895
|
+
});
|
|
70896
|
+
});
|
|
70897
|
+
var perCol = document.getElementById('ii-asof-percol');
|
|
70898
|
+
if (perCol) perCol.addEventListener('change', function () {
|
|
70899
|
+
var row = document.getElementById('ii-asof-col-row');
|
|
70900
|
+
var dateEl = document.getElementById('ii-asof');
|
|
70901
|
+
if (row) row.style.display = perCol.checked ? '' : 'none';
|
|
70902
|
+
if (dateEl) dateEl.disabled = perCol.checked;
|
|
70903
|
+
});
|
|
70904
|
+
|
|
70905
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
70906
|
+
if (applyBtn) applyBtn.addEventListener('click', function () {
|
|
70907
|
+
runInlineImport(autoImport.fileId, title, content);
|
|
70908
|
+
});
|
|
70909
|
+
}
|
|
70910
|
+
|
|
70911
|
+
// POST the confirmed proposal to /api/import/apply and stream the pipeline
|
|
70912
|
+
// live into the card's log. On 'done' show a success summary + refresh the
|
|
70913
|
+
// Objects nav in place; on 'error' show the message.
|
|
70914
|
+
function runInlineImport(fileId, title, content) {
|
|
70915
|
+
var sel = content.querySelector('input[name="ii-mode"]:checked');
|
|
70916
|
+
var mode = sel ? sel.value : 'both';
|
|
70917
|
+
var asofEl = document.getElementById('ii-asof');
|
|
70918
|
+
var asOf = asofEl ? asofEl.value : '';
|
|
70919
|
+
var perColEl = document.getElementById('ii-asof-percol');
|
|
70920
|
+
var colSel = document.getElementById('ii-asof-col');
|
|
70921
|
+
var asOfColumn = (perColEl && perColEl.checked && colSel) ? colSel.value : '';
|
|
70922
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
70923
|
+
if (applyBtn) applyBtn.disabled = true;
|
|
70924
|
+
|
|
70925
|
+
var feedEl = iiRailFeed();
|
|
70926
|
+
var log = document.getElementById('ii-log');
|
|
70927
|
+
function addLine(text, cls) {
|
|
70928
|
+
if (!log) return null;
|
|
70929
|
+
var d = document.createElement('div');
|
|
70930
|
+
d.className = 'imp-card-line' + (cls ? ' ' + cls : '');
|
|
70931
|
+
d.textContent = text;
|
|
70932
|
+
log.appendChild(d);
|
|
70933
|
+
while (log.childNodes.length > 60) log.removeChild(log.firstChild);
|
|
70934
|
+
log.scrollTop = log.scrollHeight;
|
|
70935
|
+
if (feedEl) feedEl.scrollTop = feedEl.scrollHeight;
|
|
70936
|
+
return d;
|
|
70937
|
+
}
|
|
70938
|
+
title.textContent = 'Importing your data\u2026';
|
|
70939
|
+
addLine('Starting\u2026');
|
|
70940
|
+
|
|
70941
|
+
iiStreamNdjson('/api/import/apply', { fileId: fileId, mode: mode, asOf: asOf, asOfColumn: asOfColumn }, function (evt) {
|
|
70942
|
+
if (!evt) return;
|
|
70943
|
+
if (evt.phase === 'done') {
|
|
70944
|
+
var r = evt.result || {};
|
|
70945
|
+
var rbt = r.rowsByTable || {};
|
|
70946
|
+
var names = Object.keys(rbt);
|
|
70947
|
+
var total = 0;
|
|
70948
|
+
names.forEach(function (n) { total += (rbt[n] || 0); });
|
|
70949
|
+
title.textContent = 'Imported ' + names.length + ' tables' + (mode === 'schema' ? '' : ', ' + total + ' rows');
|
|
70950
|
+
var upd = addLine('Updating your objects\u2026', 'imp-spin');
|
|
70951
|
+
refreshEntities().then(function () {
|
|
70952
|
+
renderSidebar();
|
|
70953
|
+
renderRoute();
|
|
70954
|
+
var count = (state.entities && state.entities.tables) ? state.entities.tables.length : names.length;
|
|
70955
|
+
if (upd) {
|
|
70956
|
+
upd.className = 'imp-card-line imp-done';
|
|
70957
|
+
upd.textContent = '\u2713 Done \u2014 ' + count + ' objects in your workspace';
|
|
70958
|
+
}
|
|
70959
|
+
}).catch(function () {
|
|
70960
|
+
if (upd) {
|
|
70961
|
+
upd.className = 'imp-card-line imp-err';
|
|
70962
|
+
upd.textContent = 'Imported, but refreshing the view failed \u2014 reload to see your objects.';
|
|
70963
|
+
}
|
|
70964
|
+
});
|
|
70965
|
+
} else if (evt.phase === 'error') {
|
|
70966
|
+
title.textContent = 'Import failed';
|
|
70967
|
+
addLine('Error: ' + (evt.message || 'import failed'), 'imp-err');
|
|
70968
|
+
} else if (evt.message) {
|
|
70969
|
+
addLine(evt.message);
|
|
70970
|
+
}
|
|
70971
|
+
});
|
|
70972
|
+
}
|
|
70973
|
+
`;
|
|
70974
|
+
|
|
70547
70975
|
// src/gui/app/modules/index.ts
|
|
70548
70976
|
var appJs = [
|
|
70549
70977
|
displayConfigJs,
|
|
@@ -70572,7 +71000,8 @@ var appJs = [
|
|
|
70572
71000
|
dataModelJs,
|
|
70573
71001
|
latticeTeamsJs,
|
|
70574
71002
|
onboardingJs,
|
|
70575
|
-
createDatabaseWizardJs
|
|
71003
|
+
createDatabaseWizardJs,
|
|
71004
|
+
inlineImportJs
|
|
70576
71005
|
].join("");
|
|
70577
71006
|
|
|
70578
71007
|
// src/gui/app/analytics.ts
|
|
@@ -72952,14 +73381,1167 @@ init_extract();
|
|
|
72952
73381
|
import { statSync as statSync9 } from "fs";
|
|
72953
73382
|
import { writeFile as writeFile2, rm } from "fs/promises";
|
|
72954
73383
|
import { tmpdir as tmpdir2 } from "os";
|
|
72955
|
-
import { basename as
|
|
73384
|
+
import { basename as basename12, extname as extname2, resolve as resolve11, join as join29 } from "path";
|
|
72956
73385
|
init_assistant_routes();
|
|
72957
73386
|
init_http2();
|
|
72958
73387
|
init_enrich();
|
|
72959
73388
|
init_ingest_url();
|
|
72960
73389
|
init_file_row();
|
|
72961
|
-
import { createHash as
|
|
73390
|
+
import { createHash as createHash14 } from "crypto";
|
|
72962
73391
|
init_dedup_service();
|
|
73392
|
+
|
|
73393
|
+
// src/gui/import-auto.ts
|
|
73394
|
+
import { readFileSync as readFileSync23 } from "fs";
|
|
73395
|
+
|
|
73396
|
+
// src/import/infer.ts
|
|
73397
|
+
var SAMPLE = 300;
|
|
73398
|
+
var PREFERRED_KEYS = ["code", "id", "slug", "key", "ticker", "symbol"];
|
|
73399
|
+
var NEVER_KEY = /* @__PURE__ */ new Set([
|
|
73400
|
+
"description",
|
|
73401
|
+
"notes",
|
|
73402
|
+
"summary",
|
|
73403
|
+
"desc",
|
|
73404
|
+
"comment",
|
|
73405
|
+
"comments",
|
|
73406
|
+
"bio",
|
|
73407
|
+
"text",
|
|
73408
|
+
"body"
|
|
73409
|
+
]);
|
|
73410
|
+
var FREETEXT = /* @__PURE__ */ new Set([...NEVER_KEY, "name", "title", "company", "label"]);
|
|
73411
|
+
var DIM_MAX_DISTINCT = 64;
|
|
73412
|
+
var DIM_MAX_RATIO = 0.5;
|
|
73413
|
+
var LINK_MIN_CONFIDENCE = 0.3;
|
|
73414
|
+
function isPlainObject(v2) {
|
|
73415
|
+
return typeof v2 === "object" && v2 !== null && !Array.isArray(v2);
|
|
73416
|
+
}
|
|
73417
|
+
function sourceRecords(data, entity) {
|
|
73418
|
+
const v2 = data[entity.sourceKey];
|
|
73419
|
+
if (!Array.isArray(v2)) return [];
|
|
73420
|
+
if (entity.columnar) {
|
|
73421
|
+
const cols = data[entity.sourceKey + "Cols"];
|
|
73422
|
+
if (!Array.isArray(cols)) return [];
|
|
73423
|
+
return v2.map((row) => {
|
|
73424
|
+
const o3 = {};
|
|
73425
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
73426
|
+
return o3;
|
|
73427
|
+
});
|
|
73428
|
+
}
|
|
73429
|
+
return v2.filter(isPlainObject);
|
|
73430
|
+
}
|
|
73431
|
+
function normalizeName(key) {
|
|
73432
|
+
const s2 = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
73433
|
+
if (!s2) return "field";
|
|
73434
|
+
return /^[a-z]/.test(s2) ? s2 : "f_" + s2;
|
|
73435
|
+
}
|
|
73436
|
+
var ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
73437
|
+
var ISO_DATETIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/;
|
|
73438
|
+
function inferFieldType(values) {
|
|
73439
|
+
const present = values.filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
73440
|
+
if (present.length === 0) return "text";
|
|
73441
|
+
if (present.every((v2) => typeof v2 === "number")) {
|
|
73442
|
+
return present.every((v2) => Number.isInteger(v2)) ? "integer" : "real";
|
|
73443
|
+
}
|
|
73444
|
+
if (present.every((v2) => typeof v2 === "boolean")) return "boolean";
|
|
73445
|
+
if (present.every((v2) => typeof v2 === "string")) {
|
|
73446
|
+
if (present.every((v2) => ISO_DATE.test(v2))) return "date";
|
|
73447
|
+
if (present.every((v2) => ISO_DATETIME.test(v2))) return "datetime";
|
|
73448
|
+
}
|
|
73449
|
+
return "text";
|
|
73450
|
+
}
|
|
73451
|
+
function norm2(v2) {
|
|
73452
|
+
return String(v2).trim().toLowerCase();
|
|
73453
|
+
}
|
|
73454
|
+
function isNumericValue(v2) {
|
|
73455
|
+
if (typeof v2 === "number") return Number.isFinite(v2);
|
|
73456
|
+
if (typeof v2 !== "string") return false;
|
|
73457
|
+
const s2 = v2.replace(/[\s,$%()]/g, "");
|
|
73458
|
+
return s2 !== "" && Number.isFinite(Number(s2));
|
|
73459
|
+
}
|
|
73460
|
+
function profileColumns(records) {
|
|
73461
|
+
const keys = /* @__PURE__ */ new Set();
|
|
73462
|
+
for (const r6 of records.slice(0, SAMPLE)) for (const k6 of Object.keys(r6)) keys.add(k6);
|
|
73463
|
+
const out = /* @__PURE__ */ new Map();
|
|
73464
|
+
for (const key of keys) {
|
|
73465
|
+
let isArray = false;
|
|
73466
|
+
const sample = [];
|
|
73467
|
+
const valueSet = /* @__PURE__ */ new Set();
|
|
73468
|
+
const distinctSet = /* @__PURE__ */ new Set();
|
|
73469
|
+
let nonNull = 0;
|
|
73470
|
+
let numeric = 0;
|
|
73471
|
+
for (const r6 of records) {
|
|
73472
|
+
const v2 = r6[key];
|
|
73473
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
73474
|
+
nonNull++;
|
|
73475
|
+
if (Array.isArray(v2)) {
|
|
73476
|
+
isArray = true;
|
|
73477
|
+
for (const e6 of v2) {
|
|
73478
|
+
if (e6 !== null && e6 !== void 0 && e6 !== "") {
|
|
73479
|
+
valueSet.add(norm2(e6));
|
|
73480
|
+
distinctSet.add(norm2(e6));
|
|
73481
|
+
}
|
|
73482
|
+
}
|
|
73483
|
+
} else {
|
|
73484
|
+
if (sample.length < SAMPLE) sample.push(v2);
|
|
73485
|
+
if (typeof v2 === "string") valueSet.add(norm2(v2));
|
|
73486
|
+
distinctSet.add(norm2(v2));
|
|
73487
|
+
if (isNumericValue(v2)) numeric++;
|
|
73488
|
+
}
|
|
73489
|
+
}
|
|
73490
|
+
out.set(key, {
|
|
73491
|
+
sourceKey: key,
|
|
73492
|
+
isArray,
|
|
73493
|
+
type: isArray ? "text" : inferFieldType(sample),
|
|
73494
|
+
// Cardinality counts ALL distinct values (numbers + strings). Counting only
|
|
73495
|
+
// string values let a mostly-numeric column with a few text sentinels (e.g.
|
|
73496
|
+
// a "TEV/EBITDA" of numbers + "NM") look low-cardinality and slip in as a
|
|
73497
|
+
// junk dimension.
|
|
73498
|
+
distinct: distinctSet.size,
|
|
73499
|
+
valueSet,
|
|
73500
|
+
numericFraction: nonNull > 0 ? numeric / nonNull : 0
|
|
73501
|
+
});
|
|
73502
|
+
}
|
|
73503
|
+
return out;
|
|
73504
|
+
}
|
|
73505
|
+
function pickNaturalKey(records, profiles) {
|
|
73506
|
+
const n3 = records.length;
|
|
73507
|
+
const isUnique = (key) => {
|
|
73508
|
+
const seen = /* @__PURE__ */ new Set();
|
|
73509
|
+
for (const r6 of records) {
|
|
73510
|
+
const v2 = r6[key];
|
|
73511
|
+
if (v2 === null || v2 === void 0 || v2 === "") return false;
|
|
73512
|
+
const k6 = norm2(v2);
|
|
73513
|
+
if (seen.has(k6)) return false;
|
|
73514
|
+
seen.add(k6);
|
|
73515
|
+
}
|
|
73516
|
+
return seen.size === n3;
|
|
73517
|
+
};
|
|
73518
|
+
for (const pref of PREFERRED_KEYS) {
|
|
73519
|
+
for (const [key, p3] of profiles) {
|
|
73520
|
+
if (p3.isArray) continue;
|
|
73521
|
+
if (normalizeName(key) === pref && isUnique(key)) return key;
|
|
73522
|
+
}
|
|
73523
|
+
}
|
|
73524
|
+
for (const [key, p3] of profiles) {
|
|
73525
|
+
if (p3.isArray) continue;
|
|
73526
|
+
if (NEVER_KEY.has(normalizeName(key))) continue;
|
|
73527
|
+
if ((p3.type === "text" || p3.type === "integer") && isUnique(key)) return key;
|
|
73528
|
+
}
|
|
73529
|
+
return null;
|
|
73530
|
+
}
|
|
73531
|
+
function inferSchema(data, opts = {}) {
|
|
73532
|
+
const skipped = [];
|
|
73533
|
+
const consumedColsKeys = /* @__PURE__ */ new Set();
|
|
73534
|
+
for (const key of Object.keys(data)) {
|
|
73535
|
+
const v2 = data[key];
|
|
73536
|
+
const cols = data[key + "Cols"];
|
|
73537
|
+
if (Array.isArray(v2) && v2.length > 0 && Array.isArray(v2[0]) && Array.isArray(cols) && cols.every((c6) => typeof c6 === "string")) {
|
|
73538
|
+
consumedColsKeys.add(key + "Cols");
|
|
73539
|
+
}
|
|
73540
|
+
}
|
|
73541
|
+
const sources = [];
|
|
73542
|
+
for (const key of Object.keys(data)) {
|
|
73543
|
+
if (consumedColsKeys.has(key)) continue;
|
|
73544
|
+
const v2 = data[key];
|
|
73545
|
+
if (!Array.isArray(v2) || v2.length === 0) {
|
|
73546
|
+
skipped.push({
|
|
73547
|
+
key,
|
|
73548
|
+
reason: isPlainObject(v2) ? "object (derived/rollup)" : "scalar/empty (meta or derived)"
|
|
73549
|
+
});
|
|
73550
|
+
continue;
|
|
73551
|
+
}
|
|
73552
|
+
let records;
|
|
73553
|
+
let columnar = false;
|
|
73554
|
+
if (isPlainObject(v2[0])) {
|
|
73555
|
+
records = v2.filter(isPlainObject);
|
|
73556
|
+
} else if (Array.isArray(v2[0]) && Array.isArray(data[key + "Cols"])) {
|
|
73557
|
+
const cols = data[key + "Cols"];
|
|
73558
|
+
records = v2.map((row) => {
|
|
73559
|
+
const o3 = {};
|
|
73560
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
73561
|
+
return o3;
|
|
73562
|
+
});
|
|
73563
|
+
columnar = true;
|
|
73564
|
+
} else {
|
|
73565
|
+
skipped.push({ key, reason: "array of scalars (not a record set)" });
|
|
73566
|
+
continue;
|
|
73567
|
+
}
|
|
73568
|
+
const name = opts.rename?.[key] ?? normalizeName(key);
|
|
73569
|
+
const profiles = profileColumns(records);
|
|
73570
|
+
sources.push({
|
|
73571
|
+
name,
|
|
73572
|
+
sourceKey: key,
|
|
73573
|
+
records,
|
|
73574
|
+
columnar,
|
|
73575
|
+
profiles,
|
|
73576
|
+
naturalKey: pickNaturalKey(records, profiles)
|
|
73577
|
+
});
|
|
73578
|
+
}
|
|
73579
|
+
const linkages = [];
|
|
73580
|
+
const consumedFields = /* @__PURE__ */ new Map();
|
|
73581
|
+
const linkedTargets = /* @__PURE__ */ new Map();
|
|
73582
|
+
const consume = (e6, f6) => {
|
|
73583
|
+
let set = consumedFields.get(e6);
|
|
73584
|
+
if (!set) {
|
|
73585
|
+
set = /* @__PURE__ */ new Set();
|
|
73586
|
+
consumedFields.set(e6, set);
|
|
73587
|
+
}
|
|
73588
|
+
set.add(f6);
|
|
73589
|
+
};
|
|
73590
|
+
const markTarget = (e6, t8) => {
|
|
73591
|
+
let set = linkedTargets.get(e6);
|
|
73592
|
+
if (!set) {
|
|
73593
|
+
set = /* @__PURE__ */ new Set();
|
|
73594
|
+
linkedTargets.set(e6, set);
|
|
73595
|
+
}
|
|
73596
|
+
set.add(t8);
|
|
73597
|
+
};
|
|
73598
|
+
function bestTarget(self, values) {
|
|
73599
|
+
if (values.size === 0) return null;
|
|
73600
|
+
let best = null;
|
|
73601
|
+
for (const t8 of sources) {
|
|
73602
|
+
if (t8.name === self.name || !t8.naturalKey) continue;
|
|
73603
|
+
const p3 = t8.profiles.get(t8.naturalKey);
|
|
73604
|
+
if (!p3 || p3.valueSet.size === 0) continue;
|
|
73605
|
+
let matched = 0;
|
|
73606
|
+
for (const v2 of values) if (p3.valueSet.has(v2)) matched++;
|
|
73607
|
+
if (matched > 0 && (best === null || matched > best.matched)) {
|
|
73608
|
+
best = { target: t8, column: t8.naturalKey, matched };
|
|
73609
|
+
}
|
|
73610
|
+
}
|
|
73611
|
+
return best;
|
|
73612
|
+
}
|
|
73613
|
+
for (const pass of ["array", "scalar"]) {
|
|
73614
|
+
for (const e6 of sources) {
|
|
73615
|
+
for (const [field, p3] of e6.profiles) {
|
|
73616
|
+
if (pass === "array" ? !p3.isArray : p3.isArray) continue;
|
|
73617
|
+
if (pass === "scalar") {
|
|
73618
|
+
if (field === e6.naturalKey) continue;
|
|
73619
|
+
if (FREETEXT.has(normalizeName(field)) || NEVER_KEY.has(normalizeName(field))) continue;
|
|
73620
|
+
if (p3.type !== "text") continue;
|
|
73621
|
+
}
|
|
73622
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
73623
|
+
const best = bestTarget(e6, p3.valueSet);
|
|
73624
|
+
if (!best) continue;
|
|
73625
|
+
const confidence = best.matched / p3.valueSet.size;
|
|
73626
|
+
if (confidence < LINK_MIN_CONFIDENCE) continue;
|
|
73627
|
+
if (linkedTargets.get(e6.name)?.has(best.target.name)) {
|
|
73628
|
+
consume(e6.name, field);
|
|
73629
|
+
continue;
|
|
73630
|
+
}
|
|
73631
|
+
const link = {
|
|
73632
|
+
kind: pass === "array" ? "many-to-many" : "many-to-one",
|
|
73633
|
+
fromEntity: e6.name,
|
|
73634
|
+
fromField: field,
|
|
73635
|
+
toEntity: best.target.name,
|
|
73636
|
+
toKey: normalizeName(best.column),
|
|
73637
|
+
matched: best.matched,
|
|
73638
|
+
unresolved: p3.valueSet.size - best.matched,
|
|
73639
|
+
confidence
|
|
73640
|
+
};
|
|
73641
|
+
if (pass === "array") link.junction = `${e6.name}_${best.target.name}`;
|
|
73642
|
+
linkages.push(link);
|
|
73643
|
+
consume(e6.name, field);
|
|
73644
|
+
markTarget(e6.name, best.target.name);
|
|
73645
|
+
}
|
|
73646
|
+
}
|
|
73647
|
+
}
|
|
73648
|
+
const dimColumnNames = /* @__PURE__ */ new Map();
|
|
73649
|
+
for (const e6 of sources) {
|
|
73650
|
+
for (const [field, p3] of e6.profiles) {
|
|
73651
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
73652
|
+
const nn = normalizeName(field);
|
|
73653
|
+
let arr = dimColumnNames.get(nn);
|
|
73654
|
+
if (!arr) {
|
|
73655
|
+
arr = [];
|
|
73656
|
+
dimColumnNames.set(nn, arr);
|
|
73657
|
+
}
|
|
73658
|
+
arr.push(e6);
|
|
73659
|
+
}
|
|
73660
|
+
}
|
|
73661
|
+
const dimensions = [];
|
|
73662
|
+
const dimByName = /* @__PURE__ */ new Map();
|
|
73663
|
+
for (const e6 of sources) {
|
|
73664
|
+
for (const [field, p3] of e6.profiles) {
|
|
73665
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
73666
|
+
if (field === e6.naturalKey) continue;
|
|
73667
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
73668
|
+
const nn = normalizeName(field);
|
|
73669
|
+
if (FREETEXT.has(nn)) continue;
|
|
73670
|
+
const ratio = p3.distinct / Math.max(1, e6.records.length);
|
|
73671
|
+
const sharedAcross = dimColumnNames.get(nn)?.length ?? 1;
|
|
73672
|
+
const isDim = p3.distinct >= 1 && p3.distinct <= DIM_MAX_DISTINCT && (ratio <= DIM_MAX_RATIO || sharedAcross >= 2);
|
|
73673
|
+
if (!isDim) continue;
|
|
73674
|
+
let dim = dimByName.get(nn);
|
|
73675
|
+
if (!dim) {
|
|
73676
|
+
dim = { name: nn, sourceField: field, fromEntities: [], distinctValues: 0 };
|
|
73677
|
+
dimByName.set(nn, dim);
|
|
73678
|
+
dimensions.push(dim);
|
|
73679
|
+
}
|
|
73680
|
+
if (!dim.fromEntities.includes(e6.name)) dim.fromEntities.push(e6.name);
|
|
73681
|
+
linkages.push({
|
|
73682
|
+
kind: "dimension",
|
|
73683
|
+
fromEntity: e6.name,
|
|
73684
|
+
fromField: field,
|
|
73685
|
+
toEntity: nn,
|
|
73686
|
+
toKey: "value",
|
|
73687
|
+
junction: `${e6.name}_${nn}`,
|
|
73688
|
+
matched: p3.distinct,
|
|
73689
|
+
unresolved: 0,
|
|
73690
|
+
confidence: 1
|
|
73691
|
+
});
|
|
73692
|
+
consume(e6.name, field);
|
|
73693
|
+
}
|
|
73694
|
+
}
|
|
73695
|
+
for (const dim of dimensions) {
|
|
73696
|
+
const all = /* @__PURE__ */ new Set();
|
|
73697
|
+
for (const name of dim.fromEntities) {
|
|
73698
|
+
const e6 = sources.find((s2) => s2.name === name);
|
|
73699
|
+
if (!e6) continue;
|
|
73700
|
+
for (const [f6, p3] of e6.profiles) {
|
|
73701
|
+
if (normalizeName(f6) === dim.name) for (const v2 of p3.valueSet) all.add(v2);
|
|
73702
|
+
}
|
|
73703
|
+
}
|
|
73704
|
+
dim.distinctValues = all.size;
|
|
73705
|
+
}
|
|
73706
|
+
const entities = sources.map((e6) => {
|
|
73707
|
+
const columns = [];
|
|
73708
|
+
for (const [field, p3] of e6.profiles) {
|
|
73709
|
+
if (p3.isArray) continue;
|
|
73710
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
73711
|
+
columns.push({ name: normalizeName(field), sourceKey: field, type: p3.type });
|
|
73712
|
+
}
|
|
73713
|
+
return {
|
|
73714
|
+
name: e6.name,
|
|
73715
|
+
sourceKey: e6.sourceKey,
|
|
73716
|
+
columns,
|
|
73717
|
+
naturalKey: e6.naturalKey ? normalizeName(e6.naturalKey) : null,
|
|
73718
|
+
naturalKeySource: e6.naturalKey,
|
|
73719
|
+
rowCount: e6.records.length,
|
|
73720
|
+
columnar: e6.columnar
|
|
73721
|
+
};
|
|
73722
|
+
});
|
|
73723
|
+
return { entities, dimensions, linkages, skipped };
|
|
73724
|
+
}
|
|
73725
|
+
|
|
73726
|
+
// src/import/dedupe-views.ts
|
|
73727
|
+
init_normalize();
|
|
73728
|
+
var SAMPLE2 = 300;
|
|
73729
|
+
var VIEW_MIN_OVERLAP = 0.8;
|
|
73730
|
+
function buildEntityData(plan, data) {
|
|
73731
|
+
return plan.entities.map((e6) => {
|
|
73732
|
+
const records = sourceRecords(data, e6);
|
|
73733
|
+
const colSet = /* @__PURE__ */ new Set();
|
|
73734
|
+
const colSource = /* @__PURE__ */ new Map();
|
|
73735
|
+
for (const r6 of records.slice(0, SAMPLE2)) {
|
|
73736
|
+
for (const k6 of Object.keys(r6)) {
|
|
73737
|
+
const n3 = normalizeName(k6);
|
|
73738
|
+
colSet.add(n3);
|
|
73739
|
+
if (!colSource.has(n3)) colSource.set(n3, k6);
|
|
73740
|
+
}
|
|
73741
|
+
}
|
|
73742
|
+
const normRows = records.map((r6) => {
|
|
73743
|
+
const o3 = {};
|
|
73744
|
+
for (const k6 of Object.keys(r6)) o3[normalizeName(k6)] = r6[k6];
|
|
73745
|
+
return o3;
|
|
73746
|
+
});
|
|
73747
|
+
return { name: e6.name, sourceKey: e6.sourceKey, cols: [...colSet], colSource, normRows };
|
|
73748
|
+
});
|
|
73749
|
+
}
|
|
73750
|
+
function pickIdentity(a6, shared) {
|
|
73751
|
+
let bestCol = null;
|
|
73752
|
+
let bestDistinct = -1;
|
|
73753
|
+
for (const c6 of shared) {
|
|
73754
|
+
const vals = /* @__PURE__ */ new Set();
|
|
73755
|
+
let textish = 0;
|
|
73756
|
+
let total = 0;
|
|
73757
|
+
for (const r6 of a6.normRows) {
|
|
73758
|
+
const v2 = r6[c6];
|
|
73759
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
73760
|
+
total++;
|
|
73761
|
+
if (typeof v2 === "string") textish++;
|
|
73762
|
+
vals.add(normalizeText(v2));
|
|
73763
|
+
}
|
|
73764
|
+
if (total === 0 || textish / total < 0.7) continue;
|
|
73765
|
+
if (vals.size > bestDistinct) {
|
|
73766
|
+
bestDistinct = vals.size;
|
|
73767
|
+
bestCol = c6;
|
|
73768
|
+
}
|
|
73769
|
+
}
|
|
73770
|
+
return bestCol;
|
|
73771
|
+
}
|
|
73772
|
+
function dedupeAndDetectViews(plan, data) {
|
|
73773
|
+
const entities = buildEntityData(plan, data);
|
|
73774
|
+
const views = [];
|
|
73775
|
+
const asView = /* @__PURE__ */ new Set();
|
|
73776
|
+
const colKeeps = [];
|
|
73777
|
+
for (const a6 of entities) {
|
|
73778
|
+
if (a6.cols.length < 2 || a6.normRows.length === 0) continue;
|
|
73779
|
+
const tabName = normalizeText(a6.sourceKey);
|
|
73780
|
+
if (!tabName) continue;
|
|
73781
|
+
const aColSet = new Set(a6.cols);
|
|
73782
|
+
let best = null;
|
|
73783
|
+
for (const b6 of entities) {
|
|
73784
|
+
if (b6.name === a6.name || asView.has(b6.name)) continue;
|
|
73785
|
+
if (b6.normRows.length < a6.normRows.length) continue;
|
|
73786
|
+
const bColSet = new Set(b6.cols);
|
|
73787
|
+
const shared = a6.cols.filter((c6) => bColSet.has(c6));
|
|
73788
|
+
if (shared.length < Math.max(2, Math.ceil(a6.cols.length * 0.5))) continue;
|
|
73789
|
+
const identity = pickIdentity(a6, shared);
|
|
73790
|
+
if (!identity) continue;
|
|
73791
|
+
const aIds = new Set(
|
|
73792
|
+
a6.normRows.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== "")
|
|
73793
|
+
);
|
|
73794
|
+
if (aIds.size === 0) continue;
|
|
73795
|
+
for (const disc of b6.cols) {
|
|
73796
|
+
if (aColSet.has(disc)) continue;
|
|
73797
|
+
const sub = b6.normRows.filter((r6) => normalizeText(r6[disc]) === tabName);
|
|
73798
|
+
if (sub.length === 0) continue;
|
|
73799
|
+
const bIds = new Set(sub.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== ""));
|
|
73800
|
+
let inter = 0;
|
|
73801
|
+
for (const id of aIds) if (bIds.has(id)) inter++;
|
|
73802
|
+
const overlap = inter / aIds.size;
|
|
73803
|
+
if (overlap < VIEW_MIN_OVERLAP) continue;
|
|
73804
|
+
const rawRow = sub.find((r6) => typeof r6[disc] === "string" || typeof r6[disc] === "number");
|
|
73805
|
+
const raw = rawRow ? rawRow[disc] : void 0;
|
|
73806
|
+
if (typeof raw !== "string" && typeof raw !== "number") continue;
|
|
73807
|
+
if (best === null || overlap > best.overlap || overlap === best.overlap && b6.cols.length > best.master.cols.length) {
|
|
73808
|
+
best = { master: b6, disc, value: String(raw), matched: sub.length, overlap };
|
|
73809
|
+
}
|
|
73810
|
+
}
|
|
73811
|
+
}
|
|
73812
|
+
if (!best) continue;
|
|
73813
|
+
views.push({
|
|
73814
|
+
name: a6.name,
|
|
73815
|
+
master: best.master.name,
|
|
73816
|
+
filterColumn: best.disc,
|
|
73817
|
+
filterValue: best.value,
|
|
73818
|
+
matchedRows: best.matched
|
|
73819
|
+
});
|
|
73820
|
+
asView.add(a6.name);
|
|
73821
|
+
colKeeps.push({ master: best.master, col: best.disc });
|
|
73822
|
+
}
|
|
73823
|
+
for (const { master, col } of colKeeps) {
|
|
73824
|
+
const masterEntity = plan.entities.find((e6) => e6.name === master.name);
|
|
73825
|
+
if (!masterEntity || masterEntity.columns.some((c6) => c6.name === col)) continue;
|
|
73826
|
+
masterEntity.columns.push({
|
|
73827
|
+
name: col,
|
|
73828
|
+
sourceKey: master.colSource.get(col) ?? col,
|
|
73829
|
+
type: inferFieldType(master.normRows.map((r6) => r6[col]))
|
|
73830
|
+
});
|
|
73831
|
+
}
|
|
73832
|
+
if (views.length === 0) return { plan, views };
|
|
73833
|
+
const nextPlan = {
|
|
73834
|
+
entities: plan.entities.filter((e6) => !asView.has(e6.name)),
|
|
73835
|
+
linkages: plan.linkages.filter((l4) => !asView.has(l4.fromEntity)),
|
|
73836
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.filter((n3) => !asView.has(n3)) })).filter((d6) => d6.fromEntities.length > 0),
|
|
73837
|
+
skipped: plan.skipped
|
|
73838
|
+
};
|
|
73839
|
+
return { plan: nextPlan, views };
|
|
73840
|
+
}
|
|
73841
|
+
|
|
73842
|
+
// src/import/excel.ts
|
|
73843
|
+
import { resolve as resolve10 } from "path";
|
|
73844
|
+
var HEADER_SCAN_ROWS = 25;
|
|
73845
|
+
function cellValue(v2) {
|
|
73846
|
+
if (v2 === null || v2 === void 0) return null;
|
|
73847
|
+
if (v2 instanceof Date) return v2.toISOString().slice(0, 10);
|
|
73848
|
+
if (typeof v2 === "object") {
|
|
73849
|
+
const o3 = v2;
|
|
73850
|
+
if ("result" in o3) return cellValue(o3.result);
|
|
73851
|
+
if ("text" in o3) return o3.text;
|
|
73852
|
+
if ("richText" in o3 && Array.isArray(o3.richText)) {
|
|
73853
|
+
return o3.richText.map((t8) => t8.text ?? "").join("");
|
|
73854
|
+
}
|
|
73855
|
+
return null;
|
|
73856
|
+
}
|
|
73857
|
+
return v2;
|
|
73858
|
+
}
|
|
73859
|
+
function isFilled(v2) {
|
|
73860
|
+
return v2 !== null && v2 !== void 0 && v2 !== "";
|
|
73861
|
+
}
|
|
73862
|
+
function sheetToRecords(ws) {
|
|
73863
|
+
const rowCount = ws.rowCount;
|
|
73864
|
+
const colCount = ws.columnCount;
|
|
73865
|
+
if (rowCount < 2 || colCount < 2) return [];
|
|
73866
|
+
const nonEmpty = (r6) => {
|
|
73867
|
+
let n3 = 0;
|
|
73868
|
+
for (let c6 = 1; c6 <= colCount; c6++) if (isFilled(cellValue(ws.getCell(r6, c6).value))) n3++;
|
|
73869
|
+
return n3;
|
|
73870
|
+
};
|
|
73871
|
+
const threshold = Math.max(3, Math.floor(colCount * 0.4));
|
|
73872
|
+
let headerRow = -1;
|
|
73873
|
+
for (let r6 = 1; r6 <= Math.min(HEADER_SCAN_ROWS, rowCount); r6++) {
|
|
73874
|
+
if (nonEmpty(r6) >= threshold && r6 < rowCount && nonEmpty(r6 + 1) >= 2) {
|
|
73875
|
+
headerRow = r6;
|
|
73876
|
+
break;
|
|
73877
|
+
}
|
|
73878
|
+
}
|
|
73879
|
+
if (headerRow < 0) return [];
|
|
73880
|
+
const cols = [];
|
|
73881
|
+
const seen = /* @__PURE__ */ new Set();
|
|
73882
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
73883
|
+
const hv = cellValue(ws.getCell(headerRow, c6).value);
|
|
73884
|
+
if (!isFilled(hv)) continue;
|
|
73885
|
+
const base = String(hv).replace(/\s+/g, " ").trim();
|
|
73886
|
+
if (!base) continue;
|
|
73887
|
+
let name = base;
|
|
73888
|
+
let i6 = 2;
|
|
73889
|
+
while (seen.has(name)) name = base + " " + String(i6++);
|
|
73890
|
+
seen.add(name);
|
|
73891
|
+
cols.push({ c: c6, name });
|
|
73892
|
+
}
|
|
73893
|
+
if (cols.length === 0) return [];
|
|
73894
|
+
const records = [];
|
|
73895
|
+
for (let r6 = headerRow + 1; r6 <= rowCount; r6++) {
|
|
73896
|
+
const row = {};
|
|
73897
|
+
let any = false;
|
|
73898
|
+
for (const { c: c6, name } of cols) {
|
|
73899
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
73900
|
+
if (isFilled(v2)) {
|
|
73901
|
+
row[name] = v2;
|
|
73902
|
+
any = true;
|
|
73903
|
+
}
|
|
73904
|
+
}
|
|
73905
|
+
if (!any) break;
|
|
73906
|
+
const first = cols[0] ? row[cols[0].name] : void 0;
|
|
73907
|
+
if (typeof first === "string" && /^total\b/i.test(first.trim())) continue;
|
|
73908
|
+
records.push(row);
|
|
73909
|
+
}
|
|
73910
|
+
return records;
|
|
73911
|
+
}
|
|
73912
|
+
var preambleCache = /* @__PURE__ */ new Map();
|
|
73913
|
+
function excelPreambleText(absPath) {
|
|
73914
|
+
return preambleCache.get(resolve10(absPath)) ?? "";
|
|
73915
|
+
}
|
|
73916
|
+
function sheetPreamble(ws) {
|
|
73917
|
+
const lines = [];
|
|
73918
|
+
const rowCount = Math.min(10, ws.rowCount);
|
|
73919
|
+
const colCount = Math.min(8, ws.columnCount);
|
|
73920
|
+
for (let r6 = 1; r6 <= rowCount; r6++) {
|
|
73921
|
+
const cells = [];
|
|
73922
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
73923
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
73924
|
+
if (isFilled(v2)) cells.push(String(v2));
|
|
73925
|
+
}
|
|
73926
|
+
if (cells.length) lines.push(cells.join(" "));
|
|
73927
|
+
}
|
|
73928
|
+
return lines.join("\n");
|
|
73929
|
+
}
|
|
73930
|
+
async function excelToRecords(absPath) {
|
|
73931
|
+
let mod;
|
|
73932
|
+
try {
|
|
73933
|
+
mod = await import("exceljs");
|
|
73934
|
+
} catch {
|
|
73935
|
+
throw new Error(
|
|
73936
|
+
'Reading Excel files needs the "exceljs" package \u2014 install it with: npm install exceljs'
|
|
73937
|
+
);
|
|
73938
|
+
}
|
|
73939
|
+
const ExcelJS = mod.default ?? mod;
|
|
73940
|
+
const wb = new ExcelJS.Workbook();
|
|
73941
|
+
await wb.xlsx.readFile(absPath);
|
|
73942
|
+
const out = {};
|
|
73943
|
+
const preamble = [];
|
|
73944
|
+
const props = wb.properties;
|
|
73945
|
+
if (props?.title) preamble.push(props.title);
|
|
73946
|
+
for (const ws of wb.worksheets) {
|
|
73947
|
+
preamble.push(ws.name, sheetPreamble(ws));
|
|
73948
|
+
const records = sheetToRecords(ws);
|
|
73949
|
+
if (records.length > 0) out[ws.name] = records;
|
|
73950
|
+
}
|
|
73951
|
+
preambleCache.set(resolve10(absPath), preamble.filter(Boolean).join("\n"));
|
|
73952
|
+
return out;
|
|
73953
|
+
}
|
|
73954
|
+
|
|
73955
|
+
// src/import/match.ts
|
|
73956
|
+
var BOOKKEEPING = /* @__PURE__ */ new Set(["id", "as_of", "content_key", "deleted_at"]);
|
|
73957
|
+
var MATCH_THRESHOLD = 0.6;
|
|
73958
|
+
function signature(columns) {
|
|
73959
|
+
const out = /* @__PURE__ */ new Set();
|
|
73960
|
+
for (const c6 of columns) {
|
|
73961
|
+
const n3 = normalizeName(c6);
|
|
73962
|
+
if (!n3 || BOOKKEEPING.has(n3) || n3.endsWith("_id")) continue;
|
|
73963
|
+
out.add(n3);
|
|
73964
|
+
}
|
|
73965
|
+
return out;
|
|
73966
|
+
}
|
|
73967
|
+
function containment(a6, b6) {
|
|
73968
|
+
if (a6.size === 0) return 0;
|
|
73969
|
+
let hit = 0;
|
|
73970
|
+
for (const c6 of a6) if (b6.has(c6)) hit++;
|
|
73971
|
+
return hit / a6.size;
|
|
73972
|
+
}
|
|
73973
|
+
function matchSchemaToExisting(existing, plan) {
|
|
73974
|
+
const ex = existing.map((t8) => ({ name: t8.name, sig: signature(t8.columns) }));
|
|
73975
|
+
const matches = [];
|
|
73976
|
+
const rename = {};
|
|
73977
|
+
for (const ent of plan.entities) {
|
|
73978
|
+
const sig = signature(ent.columns.map((c6) => c6.name));
|
|
73979
|
+
if (sig.size === 0) continue;
|
|
73980
|
+
let best = null;
|
|
73981
|
+
for (const t8 of ex) {
|
|
73982
|
+
if (normalizeName(t8.name) === normalizeName(ent.name)) {
|
|
73983
|
+
best = { name: t8.name, overlap: 1 };
|
|
73984
|
+
break;
|
|
73985
|
+
}
|
|
73986
|
+
const overlap = containment(sig, t8.sig);
|
|
73987
|
+
if (overlap > (best?.overlap ?? 0)) best = { name: t8.name, overlap };
|
|
73988
|
+
}
|
|
73989
|
+
if (best && best.overlap >= MATCH_THRESHOLD) {
|
|
73990
|
+
matches.push({ from: ent.name, to: best.name, overlap: best.overlap });
|
|
73991
|
+
if (best.name !== ent.name) rename[ent.name] = best.name;
|
|
73992
|
+
}
|
|
73993
|
+
}
|
|
73994
|
+
const totalEntities = plan.entities.length;
|
|
73995
|
+
const matchedCount = matches.length;
|
|
73996
|
+
const isKnownDocument = totalEntities > 0 && matchedCount >= Math.ceil(totalEntities / 2);
|
|
73997
|
+
return { matches, rename, matchedCount, totalEntities, isKnownDocument };
|
|
73998
|
+
}
|
|
73999
|
+
function renameEntities(plan, views, rename) {
|
|
74000
|
+
if (Object.keys(rename).length === 0) return { plan, views };
|
|
74001
|
+
const r6 = (n3) => rename[n3] ?? n3;
|
|
74002
|
+
return {
|
|
74003
|
+
plan: {
|
|
74004
|
+
...plan,
|
|
74005
|
+
entities: plan.entities.map((e6) => ({ ...e6, name: r6(e6.name) })),
|
|
74006
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.map(r6) })),
|
|
74007
|
+
linkages: plan.linkages.map((l4) => ({
|
|
74008
|
+
...l4,
|
|
74009
|
+
fromEntity: r6(l4.fromEntity),
|
|
74010
|
+
toEntity: r6(l4.toEntity),
|
|
74011
|
+
...l4.junction ? { junction: l4.junction } : {}
|
|
74012
|
+
}))
|
|
74013
|
+
},
|
|
74014
|
+
views: views.map((v2) => ({ ...v2, name: r6(v2.name), master: r6(v2.master) }))
|
|
74015
|
+
};
|
|
74016
|
+
}
|
|
74017
|
+
|
|
74018
|
+
// src/import/materialize.ts
|
|
74019
|
+
init_parser();
|
|
74020
|
+
import { createHash as createHash13 } from "crypto";
|
|
74021
|
+
import { existsSync as existsSync25 } from "fs";
|
|
74022
|
+
init_normalize();
|
|
74023
|
+
|
|
74024
|
+
// src/import/asof.ts
|
|
74025
|
+
var MONTHS2 = {
|
|
74026
|
+
jan: 1,
|
|
74027
|
+
january: 1,
|
|
74028
|
+
feb: 2,
|
|
74029
|
+
february: 2,
|
|
74030
|
+
mar: 3,
|
|
74031
|
+
march: 3,
|
|
74032
|
+
apr: 4,
|
|
74033
|
+
april: 4,
|
|
74034
|
+
may: 5,
|
|
74035
|
+
jun: 6,
|
|
74036
|
+
june: 6,
|
|
74037
|
+
jul: 7,
|
|
74038
|
+
july: 7,
|
|
74039
|
+
aug: 8,
|
|
74040
|
+
august: 8,
|
|
74041
|
+
sep: 9,
|
|
74042
|
+
sept: 9,
|
|
74043
|
+
september: 9,
|
|
74044
|
+
oct: 10,
|
|
74045
|
+
october: 10,
|
|
74046
|
+
nov: 11,
|
|
74047
|
+
november: 11,
|
|
74048
|
+
dec: 12,
|
|
74049
|
+
december: 12
|
|
74050
|
+
};
|
|
74051
|
+
var ASOF_KEYWORDS = /\b(as[ -]?of|as at|period (?:end(?:ed|ing)?|of)|fye|fiscal year end(?:ed|ing)?|year[ -]?end(?:ed|ing)?|quarter[ -]?end(?:ed|ing)?|valuation date|report(?:ing)? date|effective date|dated)\b/i;
|
|
74052
|
+
function isoFrom(y2, m4, d6) {
|
|
74053
|
+
if (m4 < 1 || m4 > 12 || d6 < 1 || d6 > 31) return null;
|
|
74054
|
+
if (y2 < 2010 || y2 > 2099) return null;
|
|
74055
|
+
return `${String(y2)}-${String(m4).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
74056
|
+
}
|
|
74057
|
+
function findDates(text) {
|
|
74058
|
+
const hits = [];
|
|
74059
|
+
const push = (date2, match, index) => {
|
|
74060
|
+
if (date2) hits.push({ date: date2, match, index });
|
|
74061
|
+
};
|
|
74062
|
+
for (const m4 of text.matchAll(/(20\d{2})[-._/](\d{1,2})[-._/](\d{1,2})/g)) {
|
|
74063
|
+
push(isoFrom(Number(m4[1]), Number(m4[2]), Number(m4[3])), m4[0], m4.index);
|
|
74064
|
+
}
|
|
74065
|
+
for (const m4 of text.matchAll(/(\d{1,2})[-._/](\d{1,2})[-._/](\d{2,4})/g)) {
|
|
74066
|
+
let y2 = Number(m4[3]);
|
|
74067
|
+
if (y2 < 100) y2 += 2e3;
|
|
74068
|
+
push(isoFrom(y2, Number(m4[1]), Number(m4[2])), m4[0], m4.index);
|
|
74069
|
+
}
|
|
74070
|
+
for (const m4 of text.matchAll(/([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(20\d{2})/g)) {
|
|
74071
|
+
const mon = MONTHS2[(m4[1] ?? "").toLowerCase()];
|
|
74072
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[2])), m4[0], m4.index);
|
|
74073
|
+
}
|
|
74074
|
+
for (const m4 of text.matchAll(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(20\d{2})/g)) {
|
|
74075
|
+
const mon = MONTHS2[(m4[2] ?? "").toLowerCase()];
|
|
74076
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[1])), m4[0], m4.index);
|
|
74077
|
+
}
|
|
74078
|
+
return hits;
|
|
74079
|
+
}
|
|
74080
|
+
function parseCellDate(value) {
|
|
74081
|
+
if (value instanceof Date) {
|
|
74082
|
+
return isoFrom(value.getUTCFullYear(), value.getUTCMonth() + 1, value.getUTCDate());
|
|
74083
|
+
}
|
|
74084
|
+
if (typeof value === "string") return findDates(value)[0]?.date ?? null;
|
|
74085
|
+
return null;
|
|
74086
|
+
}
|
|
74087
|
+
function scanText(text, label) {
|
|
74088
|
+
if (!text) return [];
|
|
74089
|
+
const out = [];
|
|
74090
|
+
for (const hit of findDates(text)) {
|
|
74091
|
+
const before = text.slice(Math.max(0, hit.index - 40), hit.index);
|
|
74092
|
+
const keyworded = ASOF_KEYWORDS.test(before) || ASOF_KEYWORDS.test(hit.match);
|
|
74093
|
+
const snippet = text.slice(Math.max(0, hit.index - 24), hit.index + hit.match.length + 4).replace(/\s+/g, " ").trim();
|
|
74094
|
+
out.push({
|
|
74095
|
+
date: hit.date,
|
|
74096
|
+
source: "content",
|
|
74097
|
+
confidence: keyworded ? 0.95 : 0.7,
|
|
74098
|
+
evidence: `${label}: "${snippet}"`
|
|
74099
|
+
});
|
|
74100
|
+
}
|
|
74101
|
+
return out;
|
|
74102
|
+
}
|
|
74103
|
+
function scanFilename(fileName) {
|
|
74104
|
+
if (!fileName) return [];
|
|
74105
|
+
const base = fileName.replace(/\.[A-Za-z0-9]+$/, "");
|
|
74106
|
+
return findDates(base).map((hit, i6, all) => ({
|
|
74107
|
+
date: hit.date,
|
|
74108
|
+
source: "filename",
|
|
74109
|
+
confidence: i6 === all.length - 1 ? 0.6 : 0.45,
|
|
74110
|
+
evidence: `file name: "${hit.match}"`
|
|
74111
|
+
}));
|
|
74112
|
+
}
|
|
74113
|
+
function detectAsOfCandidates(inputs) {
|
|
74114
|
+
const all = [];
|
|
74115
|
+
for (const t8 of inputs.texts ?? []) all.push(...scanText(t8.text, t8.label));
|
|
74116
|
+
if (inputs.fileName) all.push(...scanFilename(inputs.fileName));
|
|
74117
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
74118
|
+
for (const c6 of all) {
|
|
74119
|
+
const prev = byDate.get(c6.date);
|
|
74120
|
+
if (!prev || c6.confidence > prev.confidence) byDate.set(c6.date, c6);
|
|
74121
|
+
}
|
|
74122
|
+
return [...byDate.values()].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
74123
|
+
}
|
|
74124
|
+
function detectAsOf(fileName) {
|
|
74125
|
+
return scanFilename(fileName)[0]?.date ?? null;
|
|
74126
|
+
}
|
|
74127
|
+
|
|
74128
|
+
// src/import/materialize.ts
|
|
74129
|
+
function coerce2(v2, type) {
|
|
74130
|
+
if (v2 === null || v2 === void 0 || v2 === "") return null;
|
|
74131
|
+
if (type === "boolean") return v2 === true || v2 === "true" || v2 === 1 ? 1 : 0;
|
|
74132
|
+
return v2;
|
|
74133
|
+
}
|
|
74134
|
+
function contentKey(record) {
|
|
74135
|
+
const parts = Object.keys(record).sort().map((k6) => k6 + "=" + JSON.stringify(record[k6] ?? null));
|
|
74136
|
+
return createHash13("sha256").update(parts.join("|")).digest("hex");
|
|
74137
|
+
}
|
|
74138
|
+
function persistTable(configPath, name, fields) {
|
|
74139
|
+
if (!configPath || !existsSync25(configPath)) return;
|
|
74140
|
+
try {
|
|
74141
|
+
const doc = loadConfigDoc(configPath);
|
|
74142
|
+
doc.setIn(["entities", name], { fields, outputFile: name.toUpperCase() + ".md" });
|
|
74143
|
+
saveConfigDoc(configPath, doc);
|
|
74144
|
+
} catch {
|
|
74145
|
+
}
|
|
74146
|
+
}
|
|
74147
|
+
async function materializeImport(ctx, data, plan, views = [], opts = {}) {
|
|
74148
|
+
const { db, configPath } = ctx;
|
|
74149
|
+
const mode = opts.mode ?? "both";
|
|
74150
|
+
const doSchema = mode === "schema" || mode === "both";
|
|
74151
|
+
const doContents = mode === "contents" || mode === "both";
|
|
74152
|
+
const asOf = opts.asOf?.trim() ? opts.asOf.trim() : null;
|
|
74153
|
+
const asOfColumn = opts.asOfColumn?.trim() ? opts.asOfColumn.trim() : null;
|
|
74154
|
+
const dated = asOf !== null || asOfColumn !== null;
|
|
74155
|
+
const asOfSourceKey = (entity) => asOfColumn ? entity.columns.find((c6) => c6.name === asOfColumn)?.sourceKey ?? null : null;
|
|
74156
|
+
const rowAsOf = (entity, record) => {
|
|
74157
|
+
const sk = asOfSourceKey(entity);
|
|
74158
|
+
if (sk) {
|
|
74159
|
+
const d6 = parseCellDate(record[sk]);
|
|
74160
|
+
if (d6) return d6;
|
|
74161
|
+
}
|
|
74162
|
+
return asOf;
|
|
74163
|
+
};
|
|
74164
|
+
const recordKey = (entity, record) => {
|
|
74165
|
+
const a6 = rowAsOf(entity, record);
|
|
74166
|
+
return a6 ? contentKey({ ...record, __as_of: a6 }) : contentKey(record);
|
|
74167
|
+
};
|
|
74168
|
+
const scopedKey = (a6, keyVal) => (a6 ?? "") + "|" + normalizeText(keyVal);
|
|
74169
|
+
const report = async (p3) => {
|
|
74170
|
+
await opts.onProgress?.(p3);
|
|
74171
|
+
};
|
|
74172
|
+
const tablesCreated = [];
|
|
74173
|
+
const rowsByTable = {};
|
|
74174
|
+
const links = [];
|
|
74175
|
+
const viewResults = [];
|
|
74176
|
+
const byName = new Map(plan.entities.map((e6) => [e6.name, e6]));
|
|
74177
|
+
for (const entity of plan.entities) {
|
|
74178
|
+
const keyless = entity.naturalKey === null;
|
|
74179
|
+
const columns = { id: "TEXT PRIMARY KEY" };
|
|
74180
|
+
const fieldTypes = {};
|
|
74181
|
+
const cfgFields = { id: { type: "uuid", primaryKey: true } };
|
|
74182
|
+
for (const c6 of entity.columns) {
|
|
74183
|
+
columns[c6.name] = fieldToSqliteBaseType(c6.type);
|
|
74184
|
+
fieldTypes[c6.name] = c6.type;
|
|
74185
|
+
cfgFields[c6.name] = { type: c6.type };
|
|
74186
|
+
}
|
|
74187
|
+
const needsContentKey = keyless || dated;
|
|
74188
|
+
if (needsContentKey) {
|
|
74189
|
+
columns.content_key = "TEXT";
|
|
74190
|
+
cfgFields.content_key = { type: "text" };
|
|
74191
|
+
}
|
|
74192
|
+
if (dated) {
|
|
74193
|
+
columns.as_of = "TEXT";
|
|
74194
|
+
cfgFields.as_of = { type: "text" };
|
|
74195
|
+
}
|
|
74196
|
+
columns.deleted_at = "TEXT";
|
|
74197
|
+
cfgFields.deleted_at = { type: "text" };
|
|
74198
|
+
if (!db.getRegisteredTableNames().includes(entity.name)) tablesCreated.push(entity.name);
|
|
74199
|
+
await db.defineLate(entity.name, { columns, fieldTypes, primaryKey: "id" });
|
|
74200
|
+
persistTable(configPath, entity.name, cfgFields);
|
|
74201
|
+
await report({
|
|
74202
|
+
phase: "entities",
|
|
74203
|
+
table: entity.name,
|
|
74204
|
+
message: `Created table ${entity.name}`
|
|
74205
|
+
});
|
|
74206
|
+
if (doContents) {
|
|
74207
|
+
const records = sourceRecords(data, entity);
|
|
74208
|
+
const rows = records.map((r6) => {
|
|
74209
|
+
const row = {};
|
|
74210
|
+
for (const c6 of entity.columns) row[c6.name] = coerce2(r6[c6.sourceKey], c6.type);
|
|
74211
|
+
if (needsContentKey) row.content_key = recordKey(entity, r6);
|
|
74212
|
+
if (dated) row.as_of = rowAsOf(entity, r6);
|
|
74213
|
+
return row;
|
|
74214
|
+
});
|
|
74215
|
+
await db.seed({
|
|
74216
|
+
data: rows,
|
|
74217
|
+
table: entity.name,
|
|
74218
|
+
naturalKey: dated ? "content_key" : entity.naturalKey ?? "content_key"
|
|
74219
|
+
});
|
|
74220
|
+
const n3 = await db.count(entity.name);
|
|
74221
|
+
rowsByTable[entity.name] = n3;
|
|
74222
|
+
await report({
|
|
74223
|
+
phase: "entities",
|
|
74224
|
+
table: entity.name,
|
|
74225
|
+
count: n3,
|
|
74226
|
+
message: `Loaded ${String(n3)} rows into ${entity.name}`
|
|
74227
|
+
});
|
|
74228
|
+
}
|
|
74229
|
+
}
|
|
74230
|
+
for (const dim of plan.dimensions) {
|
|
74231
|
+
if (!db.getRegisteredTableNames().includes(dim.name)) tablesCreated.push(dim.name);
|
|
74232
|
+
await db.defineLate(dim.name, {
|
|
74233
|
+
columns: { id: "TEXT PRIMARY KEY", value: "TEXT", deleted_at: "TEXT" },
|
|
74234
|
+
fieldTypes: { value: "text" },
|
|
74235
|
+
primaryKey: "id"
|
|
74236
|
+
});
|
|
74237
|
+
persistTable(configPath, dim.name, {
|
|
74238
|
+
id: { type: "uuid", primaryKey: true },
|
|
74239
|
+
value: { type: "text" },
|
|
74240
|
+
deleted_at: { type: "text" }
|
|
74241
|
+
});
|
|
74242
|
+
if (doSchema) {
|
|
74243
|
+
const values = /* @__PURE__ */ new Map();
|
|
74244
|
+
for (const ename of dim.fromEntities) {
|
|
74245
|
+
const ent = byName.get(ename);
|
|
74246
|
+
if (!ent) continue;
|
|
74247
|
+
const records = sourceRecords(data, ent);
|
|
74248
|
+
const first = records[0];
|
|
74249
|
+
const srcKey = first ? Object.keys(first).find((k6) => normalizeName(k6) === dim.name) : void 0;
|
|
74250
|
+
if (!srcKey) continue;
|
|
74251
|
+
for (const r6 of records) {
|
|
74252
|
+
const v2 = r6[srcKey];
|
|
74253
|
+
if (typeof v2 !== "string" && typeof v2 !== "number") continue;
|
|
74254
|
+
const key = normalizeText(v2);
|
|
74255
|
+
if (key !== "" && !values.has(key)) values.set(key, String(v2));
|
|
74256
|
+
}
|
|
74257
|
+
}
|
|
74258
|
+
await db.seed({
|
|
74259
|
+
data: [...values.values()].map((value) => ({ value })),
|
|
74260
|
+
table: dim.name,
|
|
74261
|
+
naturalKey: "value"
|
|
74262
|
+
});
|
|
74263
|
+
const n3 = await db.count(dim.name);
|
|
74264
|
+
rowsByTable[dim.name] = n3;
|
|
74265
|
+
await report({
|
|
74266
|
+
phase: "dimensions",
|
|
74267
|
+
table: dim.name,
|
|
74268
|
+
count: n3,
|
|
74269
|
+
message: `Dimension ${dim.name}: ${String(n3)} values`
|
|
74270
|
+
});
|
|
74271
|
+
}
|
|
74272
|
+
}
|
|
74273
|
+
const idMapCache = /* @__PURE__ */ new Map();
|
|
74274
|
+
async function idMap(table, keyCol, datedTarget) {
|
|
74275
|
+
const cacheKey = table + ":" + keyCol + ":" + (datedTarget ? "D" : "");
|
|
74276
|
+
const cached = idMapCache.get(cacheKey);
|
|
74277
|
+
if (cached) return cached;
|
|
74278
|
+
const map = /* @__PURE__ */ new Map();
|
|
74279
|
+
for (const r6 of await db.query(table)) {
|
|
74280
|
+
const k6 = r6[keyCol];
|
|
74281
|
+
if (k6 === null || k6 === void 0) continue;
|
|
74282
|
+
const mapKey = datedTarget ? scopedKey(r6.as_of, k6) : normalizeText(k6);
|
|
74283
|
+
map.set(mapKey, String(r6.id));
|
|
74284
|
+
}
|
|
74285
|
+
idMapCache.set(cacheKey, map);
|
|
74286
|
+
return map;
|
|
74287
|
+
}
|
|
74288
|
+
for (const link of plan.linkages) {
|
|
74289
|
+
const from = byName.get(link.fromEntity);
|
|
74290
|
+
if (!from) continue;
|
|
74291
|
+
const jName = link.junction ?? `${link.fromEntity}_${link.toEntity}`;
|
|
74292
|
+
const fromFk = `${link.fromEntity}_id`;
|
|
74293
|
+
const toFk = `${link.toEntity}_id`;
|
|
74294
|
+
const jCols = {
|
|
74295
|
+
id: "TEXT PRIMARY KEY",
|
|
74296
|
+
[fromFk]: "TEXT",
|
|
74297
|
+
[toFk]: "TEXT"
|
|
74298
|
+
};
|
|
74299
|
+
const jCfg = {
|
|
74300
|
+
id: { type: "uuid", primaryKey: true },
|
|
74301
|
+
[fromFk]: { type: "uuid", ref: link.fromEntity },
|
|
74302
|
+
[toFk]: { type: "uuid", ref: link.toEntity }
|
|
74303
|
+
};
|
|
74304
|
+
if (dated) {
|
|
74305
|
+
jCols.as_of = "TEXT";
|
|
74306
|
+
jCfg.as_of = { type: "text" };
|
|
74307
|
+
}
|
|
74308
|
+
if (!db.getRegisteredTableNames().includes(jName)) tablesCreated.push(jName);
|
|
74309
|
+
await db.defineLate(jName, { columns: jCols, primaryKey: "id" });
|
|
74310
|
+
persistTable(configPath, jName, jCfg);
|
|
74311
|
+
if (!doContents) continue;
|
|
74312
|
+
const fromKeyCol = from.naturalKey ?? "content_key";
|
|
74313
|
+
const toIsEntity = byName.has(link.toEntity);
|
|
74314
|
+
const fromMap = await idMap(link.fromEntity, fromKeyCol, dated);
|
|
74315
|
+
const toMap = await idMap(link.toEntity, link.toKey, toIsEntity && dated);
|
|
74316
|
+
const seen = /* @__PURE__ */ new Set();
|
|
74317
|
+
for (const r6 of await db.query(jName)) {
|
|
74318
|
+
seen.add(String(r6[fromFk]) + "|" + String(r6[toFk]));
|
|
74319
|
+
}
|
|
74320
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
74321
|
+
let created = 0;
|
|
74322
|
+
for (const record of sourceRecords(data, from)) {
|
|
74323
|
+
const a6 = rowAsOf(from, record);
|
|
74324
|
+
const fromKeyVal = from.naturalKey === null ? recordKey(from, record) : record[from.naturalKeySource ?? ""];
|
|
74325
|
+
const fromId = fromMap.get(dated ? scopedKey(a6, fromKeyVal) : normalizeText(fromKeyVal));
|
|
74326
|
+
if (!fromId) continue;
|
|
74327
|
+
const raw = record[link.fromField];
|
|
74328
|
+
const refs = Array.isArray(raw) ? raw : [raw];
|
|
74329
|
+
for (const ref of refs) {
|
|
74330
|
+
if (ref === null || ref === void 0 || ref === "") continue;
|
|
74331
|
+
const toId = toMap.get(toIsEntity && dated ? scopedKey(a6, ref) : normalizeText(ref));
|
|
74332
|
+
if (!toId) {
|
|
74333
|
+
unresolved.add(normalizeText(ref));
|
|
74334
|
+
continue;
|
|
74335
|
+
}
|
|
74336
|
+
const edge = fromId + "|" + toId;
|
|
74337
|
+
if (seen.has(edge)) continue;
|
|
74338
|
+
seen.add(edge);
|
|
74339
|
+
await db.insert(
|
|
74340
|
+
jName,
|
|
74341
|
+
dated ? { [fromFk]: fromId, [toFk]: toId, as_of: a6 } : { [fromFk]: fromId, [toFk]: toId }
|
|
74342
|
+
);
|
|
74343
|
+
created++;
|
|
74344
|
+
}
|
|
74345
|
+
}
|
|
74346
|
+
rowsByTable[jName] = created;
|
|
74347
|
+
links.push({ junction: jName, created, unresolved: unresolved.size });
|
|
74348
|
+
await report({
|
|
74349
|
+
phase: "links",
|
|
74350
|
+
table: jName,
|
|
74351
|
+
count: created,
|
|
74352
|
+
message: `Linked ${String(created)} ${jName}`
|
|
74353
|
+
});
|
|
74354
|
+
}
|
|
74355
|
+
if (doSchema) {
|
|
74356
|
+
for (const v2 of views) {
|
|
74357
|
+
const filt = v2.filterValue.replace(/'/g, "''");
|
|
74358
|
+
await execSql(db, `DROP VIEW IF EXISTS "${v2.name}"`);
|
|
74359
|
+
await execSql(
|
|
74360
|
+
db,
|
|
74361
|
+
`CREATE VIEW "${v2.name}" AS SELECT * FROM "${v2.master}" WHERE "${v2.filterColumn}" = '${filt}'`
|
|
74362
|
+
);
|
|
74363
|
+
const cols = await db.introspectColumns(v2.name);
|
|
74364
|
+
await db.defineLate(v2.name, {
|
|
74365
|
+
columns: Object.fromEntries(cols.map((c6) => [c6, "TEXT"])),
|
|
74366
|
+
render: () => ""
|
|
74367
|
+
});
|
|
74368
|
+
if (!tablesCreated.includes(v2.name)) tablesCreated.push(v2.name);
|
|
74369
|
+
const rows = await db.count(v2.name);
|
|
74370
|
+
rowsByTable[v2.name] = rows;
|
|
74371
|
+
viewResults.push({ name: v2.name, master: v2.master, rows });
|
|
74372
|
+
await report({
|
|
74373
|
+
phase: "views",
|
|
74374
|
+
table: v2.name,
|
|
74375
|
+
count: rows,
|
|
74376
|
+
message: `View ${v2.name}: ${String(rows)} rows`
|
|
74377
|
+
});
|
|
74378
|
+
}
|
|
74379
|
+
}
|
|
74380
|
+
await report({ phase: "done", message: "Import complete" });
|
|
74381
|
+
return { mode, asOf, asOfColumn, tablesCreated, rowsByTable, links, views: viewResults };
|
|
74382
|
+
}
|
|
74383
|
+
|
|
74384
|
+
// src/gui/import-auto.ts
|
|
74385
|
+
init_native_entities();
|
|
74386
|
+
|
|
74387
|
+
// src/gui/import-detect.ts
|
|
74388
|
+
import { basename as basename11 } from "path";
|
|
74389
|
+
|
|
74390
|
+
// src/gui/ai/asof-llm.ts
|
|
74391
|
+
init_assistant_routes();
|
|
74392
|
+
init_chat();
|
|
74393
|
+
var MAX_CHARS = 6e3;
|
|
74394
|
+
var SYSTEM = 'You extract the single "as of" / report / snapshot / period-end date from the text of a data file (a financial statement, track record, export, etc.). Reply with ONLY that date as ISO YYYY-MM-DD, or the exact word NONE if the text has no such date. Output nothing else \u2014 no prose, no quotes.';
|
|
74395
|
+
function parseLlmDate(reply) {
|
|
74396
|
+
if (!reply) return null;
|
|
74397
|
+
const m4 = /(20\d{2})-(\d{2})-(\d{2})/.exec(reply);
|
|
74398
|
+
if (!m4) return null;
|
|
74399
|
+
const y2 = Number(m4[1]);
|
|
74400
|
+
const mo = Number(m4[2]);
|
|
74401
|
+
const d6 = Number(m4[3]);
|
|
74402
|
+
if (mo < 1 || mo > 12 || d6 < 1 || d6 > 31 || y2 < 2010 || y2 > 2099) return null;
|
|
74403
|
+
return `${String(y2)}-${String(mo).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
74404
|
+
}
|
|
74405
|
+
async function asOfFromLlm(db, text) {
|
|
74406
|
+
const trimmed = text.trim();
|
|
74407
|
+
if (!trimmed) return null;
|
|
74408
|
+
try {
|
|
74409
|
+
const auth = await resolveClaudeAuth(db);
|
|
74410
|
+
if (!auth) return null;
|
|
74411
|
+
const client = createAnthropicClient(auth);
|
|
74412
|
+
const result = await client.runTurn({
|
|
74413
|
+
model: DEFAULT_MODEL2,
|
|
74414
|
+
system: SYSTEM,
|
|
74415
|
+
temperature: 0,
|
|
74416
|
+
tools: [],
|
|
74417
|
+
messages: [{ role: "user", content: `File text:
|
|
74418
|
+
${trimmed.slice(0, MAX_CHARS)}` }],
|
|
74419
|
+
onText: () => {
|
|
74420
|
+
}
|
|
74421
|
+
});
|
|
74422
|
+
const date2 = parseLlmDate(result.text);
|
|
74423
|
+
return date2 ? { date: date2, source: "llm", confidence: 0.85, evidence: "Claude read the file" } : null;
|
|
74424
|
+
} catch (e6) {
|
|
74425
|
+
console.warn("[import] as-of LLM fallback failed:", e6.message);
|
|
74426
|
+
return null;
|
|
74427
|
+
}
|
|
74428
|
+
}
|
|
74429
|
+
|
|
74430
|
+
// src/gui/import-detect.ts
|
|
74431
|
+
async function detectImportAsOf(db, data, opts = {}) {
|
|
74432
|
+
const fileName = opts.fileName ?? (opts.abs ? basename11(opts.abs).replace(/^[0-9a-f]{8}-/, "") : "");
|
|
74433
|
+
const texts = [];
|
|
74434
|
+
for (const [k6, v2] of Object.entries(data)) {
|
|
74435
|
+
if (!Array.isArray(v2)) texts.push({ label: "data", text: `${k6}: ${JSON.stringify(v2)}` });
|
|
74436
|
+
}
|
|
74437
|
+
if (opts.abs && /\.xlsx?$/i.test(opts.abs)) {
|
|
74438
|
+
const pre = excelPreambleText(opts.abs);
|
|
74439
|
+
if (pre) texts.push({ label: "title", text: pre });
|
|
74440
|
+
}
|
|
74441
|
+
let candidates = detectAsOfCandidates({ fileName, texts });
|
|
74442
|
+
if (!candidates[0] || candidates[0].confidence < 0.7) {
|
|
74443
|
+
const llm = await asOfFromLlm(db, texts.map((t8) => t8.text).join("\n"));
|
|
74444
|
+
if (llm) candidates = [...candidates, llm].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
74445
|
+
}
|
|
74446
|
+
return candidates;
|
|
74447
|
+
}
|
|
74448
|
+
|
|
74449
|
+
// src/import/asof-columns.ts
|
|
74450
|
+
var STRONG_NAME = /(as[_ -]?of|as[_ -]?at|report(?:ing)?[_ -]?date|valuation[_ -]?date|effective[_ -]?date|period[_ -]?end|snapshot[_ -]?date|statement[_ -]?date|fye)/i;
|
|
74451
|
+
var WEAK_NAME = /(^|_)(date|period|quarter|asof)($|_)/i;
|
|
74452
|
+
function detectAsOfColumns(data, plan) {
|
|
74453
|
+
const out = [];
|
|
74454
|
+
for (const entity of plan.entities) {
|
|
74455
|
+
const records = sourceRecords(data, entity);
|
|
74456
|
+
if (records.length < 2) continue;
|
|
74457
|
+
for (const col of entity.columns) {
|
|
74458
|
+
const strong = STRONG_NAME.test(col.name);
|
|
74459
|
+
const weak = WEAK_NAME.test(col.name);
|
|
74460
|
+
if (!strong && !weak) continue;
|
|
74461
|
+
const vals = records.map((r6) => r6[col.sourceKey]).filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
74462
|
+
if (vals.length < Math.max(3, Math.floor(records.length * 0.5))) continue;
|
|
74463
|
+
const dates = vals.map(parseCellDate).filter((d6) => d6 !== null);
|
|
74464
|
+
if (dates.length / vals.length < 0.8) continue;
|
|
74465
|
+
const distinctDates = new Set(dates).size;
|
|
74466
|
+
const typed = col.type === "date" || col.type === "datetime";
|
|
74467
|
+
let confidence = strong ? 0.9 : 0.6;
|
|
74468
|
+
if (typed) confidence += 0.03;
|
|
74469
|
+
if (distinctDates > 1) confidence += 0.04;
|
|
74470
|
+
out.push({
|
|
74471
|
+
entity: entity.name,
|
|
74472
|
+
column: col.name,
|
|
74473
|
+
confidence: Math.min(confidence, 0.97),
|
|
74474
|
+
distinctDates,
|
|
74475
|
+
evidence: `column "${col.name}" \u2014 ${String(distinctDates)} distinct date${distinctDates === 1 ? "" : "s"} across ${String(vals.length)} rows`
|
|
74476
|
+
});
|
|
74477
|
+
}
|
|
74478
|
+
}
|
|
74479
|
+
return out.sort((a6, b6) => b6.confidence - a6.confidence);
|
|
74480
|
+
}
|
|
74481
|
+
|
|
74482
|
+
// src/gui/import-auto.ts
|
|
74483
|
+
function existingDataTables(db) {
|
|
74484
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
74485
|
+
const out = [];
|
|
74486
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
74487
|
+
if (native.has(t8)) continue;
|
|
74488
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
74489
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
74490
|
+
}
|
|
74491
|
+
return out;
|
|
74492
|
+
}
|
|
74493
|
+
async function readStructured(abs, name) {
|
|
74494
|
+
if (/\.xlsx?$/i.test(name)) return excelToRecords(abs);
|
|
74495
|
+
return JSON.parse(readFileSync23(abs, "utf8"));
|
|
74496
|
+
}
|
|
74497
|
+
async function autoImportStructured(db, configPath, abs, name) {
|
|
74498
|
+
if (!/\.(xlsx?|json)$/i.test(name)) return null;
|
|
74499
|
+
let data;
|
|
74500
|
+
try {
|
|
74501
|
+
data = await readStructured(abs, name);
|
|
74502
|
+
} catch {
|
|
74503
|
+
return null;
|
|
74504
|
+
}
|
|
74505
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
74506
|
+
inferSchema(data),
|
|
74507
|
+
data
|
|
74508
|
+
);
|
|
74509
|
+
if (inferredPlan.entities.length === 0) return null;
|
|
74510
|
+
const schemaMatch = matchSchemaToExisting(existingDataTables(db), inferredPlan);
|
|
74511
|
+
const asOfCandidates = await detectImportAsOf(db, data, { abs, fileName: name });
|
|
74512
|
+
const asOf = asOfCandidates[0]?.date ?? null;
|
|
74513
|
+
const asOfColumns = detectAsOfColumns(data, inferredPlan);
|
|
74514
|
+
const proposal = {
|
|
74515
|
+
plan: inferredPlan,
|
|
74516
|
+
views: inferredViews,
|
|
74517
|
+
asOfCandidates,
|
|
74518
|
+
asOfColumns,
|
|
74519
|
+
schemaMatch,
|
|
74520
|
+
matchedCount: schemaMatch.matchedCount,
|
|
74521
|
+
totalEntities: schemaMatch.totalEntities,
|
|
74522
|
+
tables: [],
|
|
74523
|
+
rows: 0
|
|
74524
|
+
};
|
|
74525
|
+
if (!schemaMatch.isKnownDocument) {
|
|
74526
|
+
return { imported: false, reason: "new-dataset", asOf, ...proposal };
|
|
74527
|
+
}
|
|
74528
|
+
if (!asOf) {
|
|
74529
|
+
return { imported: false, reason: "needs-confirm", asOf: null, ...proposal };
|
|
74530
|
+
}
|
|
74531
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, schemaMatch.rename);
|
|
74532
|
+
const result = await materializeImport({ db, configPath }, data, plan, views, { asOf });
|
|
74533
|
+
const rows = Object.values(result.rowsByTable).reduce((a6, b6) => a6 + b6, 0);
|
|
74534
|
+
return {
|
|
74535
|
+
imported: true,
|
|
74536
|
+
asOf,
|
|
74537
|
+
matchedCount: schemaMatch.matchedCount,
|
|
74538
|
+
totalEntities: schemaMatch.totalEntities,
|
|
74539
|
+
tables: Object.keys(result.rowsByTable),
|
|
74540
|
+
rows
|
|
74541
|
+
};
|
|
74542
|
+
}
|
|
74543
|
+
|
|
74544
|
+
// src/gui/ingest-routes.ts
|
|
72963
74545
|
var MIME_BY_EXT = {
|
|
72964
74546
|
".pdf": "application/pdf",
|
|
72965
74547
|
".png": "image/png",
|
|
@@ -73095,7 +74677,6 @@ function looksLikeUrl(s2) {
|
|
|
73095
74677
|
const t8 = s2.trim();
|
|
73096
74678
|
return /^https?:\/\/\S+$/i.test(t8) && !/\s/.test(t8);
|
|
73097
74679
|
}
|
|
73098
|
-
var MAX_INGEST_BYTES = 5e7;
|
|
73099
74680
|
function readBuffer2(req, maxBytes = MAX_INGEST_BYTES) {
|
|
73100
74681
|
return new Promise((resolve_, reject) => {
|
|
73101
74682
|
const chunks = [];
|
|
@@ -73157,9 +74738,15 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73157
74738
|
const tmp = join29(tmpdir2(), `lattice-ingest-${crypto.randomUUID()}${extname2(name2)}`);
|
|
73158
74739
|
let result;
|
|
73159
74740
|
let blob = null;
|
|
74741
|
+
let autoImport = null;
|
|
73160
74742
|
try {
|
|
73161
74743
|
await writeFile2(tmp, buf);
|
|
73162
74744
|
result = await extractSource(ctx.db, tmp, mime2, name2);
|
|
74745
|
+
try {
|
|
74746
|
+
autoImport = await autoImportStructured(ctx.db, ctx.configPath ?? null, tmp, name2);
|
|
74747
|
+
} catch (e6) {
|
|
74748
|
+
console.warn("[ingest] auto-import skipped:", e6.message);
|
|
74749
|
+
}
|
|
73163
74750
|
if (ctx.latticeRoot && !realPath && shouldRetainUploadBlob(mime2, name2)) {
|
|
73164
74751
|
try {
|
|
73165
74752
|
const meta2 = await attachBlob(tmp, ctx.latticeRoot);
|
|
@@ -73175,7 +74762,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73175
74762
|
let s3Status = null;
|
|
73176
74763
|
const s3cfg = resolveActiveS3Config(ctx.configPath);
|
|
73177
74764
|
if (s3cfg) {
|
|
73178
|
-
const sha256 = blob?.sha256 ??
|
|
74765
|
+
const sha256 = blob?.sha256 ?? createHash14("sha256").update(buf).digest("hex");
|
|
73179
74766
|
const key = s3Key(s3cfg.prefix, sha256);
|
|
73180
74767
|
try {
|
|
73181
74768
|
const store = await createS3Store(s3cfg);
|
|
@@ -73198,7 +74785,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73198
74785
|
}
|
|
73199
74786
|
}
|
|
73200
74787
|
const fileId = crypto.randomUUID();
|
|
73201
|
-
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ??
|
|
74788
|
+
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ?? createHash14("sha256").update(buf).digest("hex");
|
|
73202
74789
|
const uploadRow = {
|
|
73203
74790
|
id: fileId,
|
|
73204
74791
|
...fileIdentity(name2, fileId),
|
|
@@ -73236,6 +74823,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73236
74823
|
},
|
|
73237
74824
|
forcePrivate2 ? "private" : void 0
|
|
73238
74825
|
);
|
|
74826
|
+
if (autoImport?.reason) autoImport.fileId = id2;
|
|
73239
74827
|
try {
|
|
73240
74828
|
const dedupCtx = {
|
|
73241
74829
|
db: ctx.db,
|
|
@@ -73263,6 +74851,15 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73263
74851
|
e6 instanceof Error ? e6.message : String(e6)
|
|
73264
74852
|
);
|
|
73265
74853
|
}
|
|
74854
|
+
if (autoImport?.imported) {
|
|
74855
|
+
ctx.feed.publish({
|
|
74856
|
+
table: autoImport.tables[0] ?? "files",
|
|
74857
|
+
op: "insert",
|
|
74858
|
+
rowId: null,
|
|
74859
|
+
source: "system",
|
|
74860
|
+
summary: `Imported the ${autoImport.asOf ?? ""} snapshot of "${name2}" \u2014 ${String(autoImport.rows)} rows across ${String(autoImport.tables.length)} tables`
|
|
74861
|
+
});
|
|
74862
|
+
}
|
|
73266
74863
|
let suggestedLinks = [];
|
|
73267
74864
|
if (!result.skip) {
|
|
73268
74865
|
const links = await enrichOrFail(mctx, ctx.db, id2, result.text, name2, ctx, res, forcePrivate2);
|
|
@@ -73275,6 +74872,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73275
74872
|
id: id2,
|
|
73276
74873
|
extraction_status: result.skip ? "skipped" : "extracted",
|
|
73277
74874
|
suggestedLinks,
|
|
74875
|
+
...autoImport ? { autoImport } : {},
|
|
73278
74876
|
// Present only when S3 is enabled for this workspace. 'failed' tells the
|
|
73279
74877
|
// uploader the bytes did NOT reach the shared bucket — other members would
|
|
73280
74878
|
// 404 until it's re-uploaded — so the GUI can warn rather than imply a
|
|
@@ -73360,7 +74958,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73360
74958
|
sendJson(res, { error: "path is required" }, 400);
|
|
73361
74959
|
return true;
|
|
73362
74960
|
}
|
|
73363
|
-
const abs =
|
|
74961
|
+
const abs = resolve11(rawPath);
|
|
73364
74962
|
let size = 0;
|
|
73365
74963
|
try {
|
|
73366
74964
|
const st = statSync9(abs);
|
|
@@ -73377,7 +74975,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73377
74975
|
sendJson(res, { error: "file too large" }, 413);
|
|
73378
74976
|
return true;
|
|
73379
74977
|
}
|
|
73380
|
-
const name =
|
|
74978
|
+
const name = basename12(abs);
|
|
73381
74979
|
const mime = mimeFor(name);
|
|
73382
74980
|
const localFileId = crypto.randomUUID();
|
|
73383
74981
|
const localRow = {
|
|
@@ -73443,6 +75041,146 @@ ${err.stack ?? ""}`
|
|
|
73443
75041
|
return true;
|
|
73444
75042
|
}
|
|
73445
75043
|
|
|
75044
|
+
// src/gui/import-routes.ts
|
|
75045
|
+
init_adapter();
|
|
75046
|
+
init_http2();
|
|
75047
|
+
import { existsSync as existsSync26, readFileSync as readFileSync24, statSync as statSync10 } from "fs";
|
|
75048
|
+
import { isAbsolute as isAbsolute4, join as join30 } from "path";
|
|
75049
|
+
init_native_entities();
|
|
75050
|
+
function badRequest(message) {
|
|
75051
|
+
const e6 = new Error(message);
|
|
75052
|
+
e6.statusCode = 400;
|
|
75053
|
+
return e6;
|
|
75054
|
+
}
|
|
75055
|
+
function localPathOf2(row, latticeRoot) {
|
|
75056
|
+
if (row.ref_kind === "local_ref" && row.ref_uri) return row.ref_uri;
|
|
75057
|
+
if ((row.ref_kind === "blob" || row.ref_kind === "cloud_ref") && row.blob_path) {
|
|
75058
|
+
return isAbsolute4(row.blob_path) ? row.blob_path : latticeRoot ? join30(latticeRoot, row.blob_path) : null;
|
|
75059
|
+
}
|
|
75060
|
+
return null;
|
|
75061
|
+
}
|
|
75062
|
+
function existingDataTables2(db) {
|
|
75063
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
75064
|
+
const out = [];
|
|
75065
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
75066
|
+
if (native.has(t8)) continue;
|
|
75067
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
75068
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
75069
|
+
}
|
|
75070
|
+
return out;
|
|
75071
|
+
}
|
|
75072
|
+
async function readImportSourceFromFile(db, fileId, latticeRoot) {
|
|
75073
|
+
const row = await getAsyncOrSync(
|
|
75074
|
+
db.adapter,
|
|
75075
|
+
`SELECT "id","original_name","mime","ref_kind","ref_uri","blob_path"
|
|
75076
|
+
FROM "files" WHERE "id" = ? AND "deleted_at" IS NULL LIMIT 1`,
|
|
75077
|
+
[fileId]
|
|
75078
|
+
);
|
|
75079
|
+
if (!row) throw badRequest("Unknown import file: " + fileId);
|
|
75080
|
+
const path2 = localPathOf2(row, latticeRoot);
|
|
75081
|
+
if (!path2 || !existsSync26(path2)) {
|
|
75082
|
+
throw badRequest("The import file\u2019s bytes are not available locally.");
|
|
75083
|
+
}
|
|
75084
|
+
const sizeBytes = statSync10(path2).size;
|
|
75085
|
+
if (sizeBytes > MAX_INGEST_BYTES) {
|
|
75086
|
+
throw badRequest(
|
|
75087
|
+
`The import file is too large (${String(Math.round(sizeBytes / 1e6))} MB); the limit is ${String(Math.round(MAX_INGEST_BYTES / 1e6))} MB.`
|
|
75088
|
+
);
|
|
75089
|
+
}
|
|
75090
|
+
const name = row.original_name ?? "";
|
|
75091
|
+
const mime = row.mime ?? "";
|
|
75092
|
+
if (/\.xlsx?$/i.test(name) || mime.includes("spreadsheet") || mime.includes("excel")) {
|
|
75093
|
+
return excelToRecords(path2);
|
|
75094
|
+
}
|
|
75095
|
+
let parsed;
|
|
75096
|
+
try {
|
|
75097
|
+
parsed = JSON.parse(readFileSync24(path2, "utf8"));
|
|
75098
|
+
} catch {
|
|
75099
|
+
throw badRequest("The import file is not valid JSON.");
|
|
75100
|
+
}
|
|
75101
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
75102
|
+
throw badRequest("Expected a JSON object whose keys are record arrays.");
|
|
75103
|
+
}
|
|
75104
|
+
return parsed;
|
|
75105
|
+
}
|
|
75106
|
+
async function dispatchImportRoute(req, res, deps) {
|
|
75107
|
+
const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
|
|
75108
|
+
if (req.method !== "POST" || pathname !== "/api/import/apply") return false;
|
|
75109
|
+
const body = await readJson(req).catch(() => ({}));
|
|
75110
|
+
const fileId = typeof body.fileId === "string" ? body.fileId : "";
|
|
75111
|
+
const mode = body.mode === "schema" || body.mode === "contents" ? body.mode : "both";
|
|
75112
|
+
const asOf = typeof body.asOf === "string" && /^\d{4}-\d{2}-\d{2}$/.test(body.asOf.trim()) ? body.asOf.trim() : null;
|
|
75113
|
+
const asOfColumn = typeof body.asOfColumn === "string" && body.asOfColumn.trim() ? body.asOfColumn.trim() : null;
|
|
75114
|
+
if (!fileId) {
|
|
75115
|
+
sendJson(res, { error: "fileId is required" }, 400);
|
|
75116
|
+
return true;
|
|
75117
|
+
}
|
|
75118
|
+
res.writeHead(200, {
|
|
75119
|
+
"content-type": "application/x-ndjson; charset=utf-8",
|
|
75120
|
+
"cache-control": "no-store"
|
|
75121
|
+
});
|
|
75122
|
+
const emit = (p3) => {
|
|
75123
|
+
res.write(JSON.stringify(p3) + "\n");
|
|
75124
|
+
};
|
|
75125
|
+
try {
|
|
75126
|
+
emit({ phase: "parse", message: "Reading source\u2026" });
|
|
75127
|
+
const data = await readImportSourceFromFile(deps.db, fileId, deps.latticeRoot);
|
|
75128
|
+
emit({ phase: "infer", message: "Analyzing schema\u2026" });
|
|
75129
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
75130
|
+
inferSchema(data),
|
|
75131
|
+
data
|
|
75132
|
+
);
|
|
75133
|
+
emit({
|
|
75134
|
+
phase: "infer",
|
|
75135
|
+
message: `Found ${String(inferredPlan.entities.length)} entities, ${String(inferredPlan.dimensions.length)} dimensions, ${String(inferredPlan.linkages.length)} links`
|
|
75136
|
+
});
|
|
75137
|
+
const match = matchSchemaToExisting(existingDataTables2(deps.db), inferredPlan);
|
|
75138
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, match.rename);
|
|
75139
|
+
if (views.length > 0) {
|
|
75140
|
+
emit({
|
|
75141
|
+
phase: "detect",
|
|
75142
|
+
message: `Detected ${String(views.length)} reconstructable views (no duplicated rows)`
|
|
75143
|
+
});
|
|
75144
|
+
}
|
|
75145
|
+
if (match.isKnownDocument) {
|
|
75146
|
+
emit({
|
|
75147
|
+
phase: "detect",
|
|
75148
|
+
message: `Recognized as a new period of an existing document \u2014 ${String(match.matchedCount)} of ${String(match.totalEntities)} tables matched`
|
|
75149
|
+
});
|
|
75150
|
+
}
|
|
75151
|
+
if (asOfColumn) {
|
|
75152
|
+
emit({ phase: "infer", message: `Dating each row by its "${asOfColumn}" column` });
|
|
75153
|
+
} else if (asOf) {
|
|
75154
|
+
emit({ phase: "infer", message: `Importing as a snapshot dated ${asOf}` });
|
|
75155
|
+
}
|
|
75156
|
+
const result = await materializeImport(
|
|
75157
|
+
{ db: deps.db, configPath: deps.configPath },
|
|
75158
|
+
data,
|
|
75159
|
+
plan,
|
|
75160
|
+
views,
|
|
75161
|
+
{
|
|
75162
|
+
mode,
|
|
75163
|
+
asOf,
|
|
75164
|
+
asOfColumn,
|
|
75165
|
+
onProgress: async (p3) => {
|
|
75166
|
+
emit({ ...p3 });
|
|
75167
|
+
await new Promise((r6) => setImmediate(r6));
|
|
75168
|
+
}
|
|
75169
|
+
}
|
|
75170
|
+
);
|
|
75171
|
+
for (const t8 of result.tablesCreated) {
|
|
75172
|
+
deps.validTables.add(t8);
|
|
75173
|
+
const cols = deps.db.getRegisteredColumns(t8);
|
|
75174
|
+
if (cols && "deleted_at" in cols) deps.softDeletable.add(t8);
|
|
75175
|
+
}
|
|
75176
|
+
emit({ phase: "done", ok: true, result });
|
|
75177
|
+
} catch (e6) {
|
|
75178
|
+
emit({ phase: "error", message: e6.message });
|
|
75179
|
+
}
|
|
75180
|
+
res.end();
|
|
75181
|
+
return true;
|
|
75182
|
+
}
|
|
75183
|
+
|
|
73446
75184
|
// src/gui/read-routes.ts
|
|
73447
75185
|
init_http2();
|
|
73448
75186
|
init_data();
|
|
@@ -73731,7 +75469,13 @@ async function handleReadRoutes(req, res, ctx, deps) {
|
|
|
73731
75469
|
return true;
|
|
73732
75470
|
}
|
|
73733
75471
|
if (method === "GET" && pathname === "/api/history") {
|
|
73734
|
-
const
|
|
75472
|
+
const limitRaw = url.searchParams.get("limit");
|
|
75473
|
+
const parsedLimit = parsePageParam(limitRaw, "limit");
|
|
75474
|
+
if (parsedLimit === "invalid") {
|
|
75475
|
+
sendJson(res, { error: "limit must be a non-negative integer" }, 400);
|
|
75476
|
+
return true;
|
|
75477
|
+
}
|
|
75478
|
+
const limit = limitRaw === null ? 200 : parsedLimit;
|
|
73735
75479
|
const filterTable = url.searchParams.get("table");
|
|
73736
75480
|
const raw = await active.db.query("_lattice_gui_audit", { limit });
|
|
73737
75481
|
let entries = raw.map(parseAudit).sort((a6, b6) => b6.ts.localeCompare(a6.ts));
|
|
@@ -74775,8 +76519,8 @@ async function handleHistoryRoutes(req, res, ctx, deps) {
|
|
|
74775
76519
|
|
|
74776
76520
|
// src/gui/workspaces-routes.ts
|
|
74777
76521
|
init_http2();
|
|
74778
|
-
import { resolve as
|
|
74779
|
-
import { existsSync as
|
|
76522
|
+
import { resolve as resolve12 } from "path";
|
|
76523
|
+
import { existsSync as existsSync27, rmSync } from "fs";
|
|
74780
76524
|
init_workspace();
|
|
74781
76525
|
init_lattice_root();
|
|
74782
76526
|
init_user_config();
|
|
@@ -74784,7 +76528,7 @@ function cleanupWorkspaceFiles(root6, ws) {
|
|
|
74784
76528
|
if (!ws.configPath && ws.kind === "local") {
|
|
74785
76529
|
rmSync(workspaceDir(root6, ws.dir), { recursive: true, force: true });
|
|
74786
76530
|
} else if (ws.kind === "cloud") {
|
|
74787
|
-
if (ws.configPath &&
|
|
76531
|
+
if (ws.configPath && existsSync27(ws.configPath)) {
|
|
74788
76532
|
rmSync(ws.configPath, { force: true });
|
|
74789
76533
|
}
|
|
74790
76534
|
const labelMatch = /^\$\{LATTICE_DB:([A-Za-z0-9._-]+)\}$/.exec(ws.db.trim());
|
|
@@ -74938,7 +76682,7 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
74938
76682
|
return true;
|
|
74939
76683
|
}
|
|
74940
76684
|
const wsPaths = resolveWorkspacePaths(latticeRoot, ws);
|
|
74941
|
-
const isActive =
|
|
76685
|
+
const isActive = resolve12(active.configPath) === resolve12(wsPaths.configPath);
|
|
74942
76686
|
let switchedTo = null;
|
|
74943
76687
|
if (isActive) {
|
|
74944
76688
|
const fallback = listWorkspaces(latticeRoot).find((w2) => w2.id !== ws.id);
|
|
@@ -74987,40 +76731,40 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
74987
76731
|
|
|
74988
76732
|
// src/gui/databases-routes.ts
|
|
74989
76733
|
init_http2();
|
|
74990
|
-
import { basename as
|
|
74991
|
-
import { existsSync as
|
|
76734
|
+
import { basename as basename14, resolve as resolve14 } from "path";
|
|
76735
|
+
import { existsSync as existsSync29 } from "fs";
|
|
74992
76736
|
init_parser();
|
|
74993
76737
|
|
|
74994
76738
|
// src/gui/config-paths.ts
|
|
74995
76739
|
init_parser();
|
|
74996
|
-
import { basename as
|
|
76740
|
+
import { basename as basename13, dirname as dirname15, join as join31, resolve as resolve13 } from "path";
|
|
74997
76741
|
import {
|
|
74998
|
-
existsSync as
|
|
76742
|
+
existsSync as existsSync28,
|
|
74999
76743
|
mkdirSync as mkdirSync11,
|
|
75000
|
-
readFileSync as
|
|
76744
|
+
readFileSync as readFileSync25,
|
|
75001
76745
|
readdirSync as readdirSync8,
|
|
75002
76746
|
unlinkSync as unlinkSync5,
|
|
75003
76747
|
writeFileSync as writeFileSync9
|
|
75004
76748
|
} from "fs";
|
|
75005
76749
|
import { parseDocument as parseDocument7 } from "yaml";
|
|
75006
76750
|
function resolveOutputDirForConfig(configPath) {
|
|
75007
|
-
const base = dirname15(
|
|
76751
|
+
const base = dirname15(resolve13(configPath));
|
|
75008
76752
|
for (const dir of ["context", ".", "generated"]) {
|
|
75009
|
-
const abs =
|
|
75010
|
-
if (
|
|
76753
|
+
const abs = resolve13(base, dir);
|
|
76754
|
+
if (existsSync28(join31(abs, ".lattice", "manifest.json"))) return abs;
|
|
75011
76755
|
}
|
|
75012
|
-
return
|
|
76756
|
+
return resolve13(base, "context");
|
|
75013
76757
|
}
|
|
75014
76758
|
function friendlyConfigName(parsedName, configPath) {
|
|
75015
76759
|
if (parsedName && parsedName.trim().length > 0) return parsedName.trim();
|
|
75016
|
-
return
|
|
76760
|
+
return basename13(configPath).replace(/\.(ya?ml)$/, "");
|
|
75017
76761
|
}
|
|
75018
76762
|
function listConfigs(activeConfigPath) {
|
|
75019
76763
|
const dir = dirname15(activeConfigPath);
|
|
75020
76764
|
const entries = [];
|
|
75021
76765
|
for (const fname of readdirSync8(dir)) {
|
|
75022
76766
|
if (!fname.endsWith(".yml") && !fname.endsWith(".yaml")) continue;
|
|
75023
|
-
const full =
|
|
76767
|
+
const full = join31(dir, fname);
|
|
75024
76768
|
try {
|
|
75025
76769
|
const parsed = parseConfigFile(full);
|
|
75026
76770
|
entries.push({
|
|
@@ -75031,7 +76775,7 @@ function listConfigs(activeConfigPath) {
|
|
|
75031
76775
|
// `label` is the friendly DB name — what the user sees in the
|
|
75032
76776
|
// dropdown + settings. Falls back to the basename when unset.
|
|
75033
76777
|
label: friendlyConfigName(parsed.name, full),
|
|
75034
|
-
dbFile:
|
|
76778
|
+
dbFile: basename13(parsed.dbPath),
|
|
75035
76779
|
active: full === activeConfigPath,
|
|
75036
76780
|
// `${LATTICE_DB:...}` and postgres:// configs resolve to a
|
|
75037
76781
|
// postgres URL; everything else is a local SQLite file. This
|
|
@@ -75048,37 +76792,37 @@ function createBlankConfig(activeConfigPath, dbName) {
|
|
|
75048
76792
|
const dir = dirname15(activeConfigPath);
|
|
75049
76793
|
const slug = dbName.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
75050
76794
|
if (!slug) throw new Error("Workspace name must contain at least one alphanumeric character");
|
|
75051
|
-
const configPath =
|
|
75052
|
-
if (
|
|
76795
|
+
const configPath = join31(dir, `${slug}.config.yml`);
|
|
76796
|
+
if (existsSync28(configPath)) throw new Error(`Config already exists: ${slug}.config.yml`);
|
|
75053
76797
|
const yaml = `db: ./data/${slug}.db
|
|
75054
76798
|
|
|
75055
76799
|
entities: {}
|
|
75056
76800
|
`;
|
|
75057
76801
|
writeFileSync9(configPath, yaml, "utf8");
|
|
75058
|
-
mkdirSync11(
|
|
76802
|
+
mkdirSync11(join31(dir, "data"), { recursive: true });
|
|
75059
76803
|
return configPath;
|
|
75060
76804
|
}
|
|
75061
76805
|
function sqliteFileForConfig(configPath) {
|
|
75062
|
-
const dbVal = parseDocument7(
|
|
76806
|
+
const dbVal = parseDocument7(readFileSync25(configPath, "utf8")).get("db");
|
|
75063
76807
|
const raw = (typeof dbVal === "string" ? dbVal : "").trim();
|
|
75064
76808
|
if (!raw) return null;
|
|
75065
76809
|
if (isPostgresUrl(raw) || raw.startsWith("${LATTICE_DB:")) return null;
|
|
75066
76810
|
if (raw === ":memory:" || raw.startsWith("file:")) return null;
|
|
75067
|
-
return
|
|
76811
|
+
return resolve13(dirname15(configPath), raw);
|
|
75068
76812
|
}
|
|
75069
76813
|
function deleteDatabaseFiles(targetConfigPath) {
|
|
75070
76814
|
const sqliteFile = sqliteFileForConfig(targetConfigPath);
|
|
75071
76815
|
unlinkSync5(targetConfigPath);
|
|
75072
76816
|
let deletedDbFile = null;
|
|
75073
|
-
if (sqliteFile &&
|
|
76817
|
+
if (sqliteFile && existsSync28(sqliteFile)) {
|
|
75074
76818
|
unlinkSync5(sqliteFile);
|
|
75075
76819
|
deletedDbFile = sqliteFile;
|
|
75076
76820
|
for (const suffix of ["-wal", "-shm", "-journal"]) {
|
|
75077
76821
|
const sidecar = sqliteFile + suffix;
|
|
75078
|
-
if (
|
|
76822
|
+
if (existsSync28(sidecar)) unlinkSync5(sidecar);
|
|
75079
76823
|
}
|
|
75080
76824
|
}
|
|
75081
|
-
return { deletedConfig:
|
|
76825
|
+
return { deletedConfig: basename13(targetConfigPath), deletedDbFile };
|
|
75082
76826
|
}
|
|
75083
76827
|
|
|
75084
76828
|
// src/gui/databases-routes.ts
|
|
@@ -75094,7 +76838,7 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
75094
76838
|
sendJson(res, {
|
|
75095
76839
|
current: {
|
|
75096
76840
|
path: active.configPath,
|
|
75097
|
-
dbFile:
|
|
76841
|
+
dbFile: basename14(parsedActive.dbPath),
|
|
75098
76842
|
label: friendlyLabel,
|
|
75099
76843
|
kind
|
|
75100
76844
|
},
|
|
@@ -75108,8 +76852,8 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
75108
76852
|
sendJson(res, { error: "path must be a string" }, 400);
|
|
75109
76853
|
return true;
|
|
75110
76854
|
}
|
|
75111
|
-
const newPath =
|
|
75112
|
-
if (!
|
|
76855
|
+
const newPath = resolve14(body.path);
|
|
76856
|
+
if (!existsSync29(newPath)) {
|
|
75113
76857
|
sendJson(res, { error: `Config not found: ${newPath}` }, 400);
|
|
75114
76858
|
return true;
|
|
75115
76859
|
}
|
|
@@ -75151,16 +76895,16 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
75151
76895
|
sendJson(res, { error: "path must be a non-empty string" }, 400);
|
|
75152
76896
|
return true;
|
|
75153
76897
|
}
|
|
75154
|
-
const target =
|
|
76898
|
+
const target = resolve14(body.path);
|
|
75155
76899
|
const known = listConfigs(active.configPath);
|
|
75156
|
-
const match = known.find((c6) =>
|
|
76900
|
+
const match = known.find((c6) => resolve14(c6.path) === target);
|
|
75157
76901
|
if (!match) {
|
|
75158
76902
|
sendJson(res, { error: `Not a known database config: ${target}` }, 400);
|
|
75159
76903
|
return true;
|
|
75160
76904
|
}
|
|
75161
76905
|
let switchedTo = null;
|
|
75162
|
-
if (
|
|
75163
|
-
const fallback = known.find((c6) =>
|
|
76906
|
+
if (resolve14(active.configPath) === target) {
|
|
76907
|
+
const fallback = known.find((c6) => resolve14(c6.path) !== target);
|
|
75164
76908
|
if (!fallback) {
|
|
75165
76909
|
sendJson(
|
|
75166
76910
|
res,
|
|
@@ -75251,20 +76995,26 @@ async function listenWithPortFallback(server, startPort, host) {
|
|
|
75251
76995
|
throw new Error(`No available port found starting at ${String(startPort)}`);
|
|
75252
76996
|
}
|
|
75253
76997
|
async function startGuiServer(options) {
|
|
75254
|
-
const bootConfigPath = options.configPath ?
|
|
75255
|
-
const bootOutputDir = options.outputDir ?
|
|
76998
|
+
const bootConfigPath = options.configPath ? resolve15(options.configPath) : null;
|
|
76999
|
+
const bootOutputDir = options.outputDir ? resolve15(options.outputDir) : null;
|
|
75256
77000
|
const startPort = options.port ?? 4317;
|
|
75257
77001
|
const host = options.host ?? "127.0.0.1";
|
|
77002
|
+
const isLoopbackHost2 = host === "localhost" || host === "::1" || host.startsWith("127.");
|
|
77003
|
+
if (!isLoopbackHost2) {
|
|
77004
|
+
console.warn(
|
|
77005
|
+
`[lattice] GUI is binding to a non-loopback address (${host}); its data routes are UNAUTHENTICATED and will be reachable from the network.`
|
|
77006
|
+
);
|
|
77007
|
+
}
|
|
75258
77008
|
const autoRender = options.autoRender ?? false;
|
|
75259
77009
|
const guiVersion = options.version ?? "";
|
|
75260
77010
|
const sessionId = crypto.randomUUID();
|
|
75261
77011
|
let updateService = null;
|
|
75262
77012
|
let activeRef = bootConfigPath && bootOutputDir ? await openConfig(bootConfigPath, bootOutputDir, autoRender, options.realtimeWatchdogMs) : null;
|
|
75263
|
-
const latticeRoot = (bootConfigPath ? findLatticeRoot(dirname16(bootConfigPath)) : null) ?? (options.latticeRoot ?
|
|
77013
|
+
const latticeRoot = (bootConfigPath ? findLatticeRoot(dirname16(bootConfigPath)) : null) ?? (options.latticeRoot ? resolve15(options.latticeRoot) : null);
|
|
75264
77014
|
let currentWorkspaceId = null;
|
|
75265
77015
|
if (latticeRoot && bootConfigPath) {
|
|
75266
77016
|
const launched = listWorkspaces(latticeRoot).find(
|
|
75267
|
-
(w2) =>
|
|
77017
|
+
(w2) => resolve15(resolveWorkspacePaths(latticeRoot, w2).configPath) === resolve15(bootConfigPath)
|
|
75268
77018
|
);
|
|
75269
77019
|
if (launched) {
|
|
75270
77020
|
currentWorkspaceId = launched.id;
|
|
@@ -75609,6 +77359,22 @@ async function startGuiServer(options) {
|
|
|
75609
77359
|
});
|
|
75610
77360
|
}
|
|
75611
77361
|
},
|
|
77362
|
+
// ── Structured-source import (apply) ──
|
|
77363
|
+
// The importer is reachable only via dropping a file in the assistant
|
|
77364
|
+
// chat; this materializes the user-confirmed proposal, re-reading the
|
|
77365
|
+
// file's bytes from its `fileId` (its retained blob).
|
|
77366
|
+
{
|
|
77367
|
+
handle: async (req2, res2) => {
|
|
77368
|
+
if (!pathname.startsWith("/api/import/")) return false;
|
|
77369
|
+
return await dispatchImportRoute(req2, res2, {
|
|
77370
|
+
db: active.db,
|
|
77371
|
+
configPath: active.configPath,
|
|
77372
|
+
latticeRoot: dirname16(active.configPath),
|
|
77373
|
+
validTables: active.validTables,
|
|
77374
|
+
softDeletable: active.softDeletable
|
|
77375
|
+
});
|
|
77376
|
+
}
|
|
77377
|
+
},
|
|
75612
77378
|
// ── Files: blob serving + open-in-finder ──
|
|
75613
77379
|
{
|
|
75614
77380
|
handle: async (req2, res2) => {
|
|
@@ -75799,6 +77565,7 @@ ${e6.stack ?? ""}`
|
|
|
75799
77565
|
server,
|
|
75800
77566
|
port,
|
|
75801
77567
|
url,
|
|
77568
|
+
whenConverged: () => activeRef?.converged ?? Promise.resolve(),
|
|
75802
77569
|
close: () => new Promise((resolveClose, reject) => {
|
|
75803
77570
|
updateService?.stop();
|
|
75804
77571
|
for (const client of wss.clients) {
|
|
@@ -75825,8 +77592,8 @@ ${e6.stack ?? ""}`
|
|
|
75825
77592
|
}
|
|
75826
77593
|
|
|
75827
77594
|
// src/cloud/file-source-key-store.ts
|
|
75828
|
-
import { readFileSync as
|
|
75829
|
-
import { dirname as dirname17, resolve as
|
|
77595
|
+
import { readFileSync as readFileSync26, writeFileSync as writeFileSync10, existsSync as existsSync30, mkdirSync as mkdirSync12, renameSync as renameSync5, chmodSync as chmodSync3 } from "fs";
|
|
77596
|
+
import { dirname as dirname17, resolve as resolve16 } from "path";
|
|
75830
77597
|
import { createCipheriv as createCipheriv3, createDecipheriv as createDecipheriv3, randomBytes as randomBytes9, scryptSync as scryptSync3 } from "crypto";
|
|
75831
77598
|
var ENC_HEADER = "LATTICE-KMS-v1\n";
|
|
75832
77599
|
var SCRYPT_N = 1 << 15;
|
|
@@ -75844,7 +77611,7 @@ var FileSourceKeyStore = class {
|
|
|
75844
77611
|
if (!opts.path || typeof opts.path !== "string") {
|
|
75845
77612
|
throw new Error("lattice: FileSourceKeyStore requires a non-empty `path`");
|
|
75846
77613
|
}
|
|
75847
|
-
this.path =
|
|
77614
|
+
this.path = resolve16(opts.path);
|
|
75848
77615
|
this.passphrase = opts.passphrase;
|
|
75849
77616
|
this.cache = this.load();
|
|
75850
77617
|
}
|
|
@@ -75877,12 +77644,12 @@ var FileSourceKeyStore = class {
|
|
|
75877
77644
|
// ── internals ────────────────────────────────────────────────────────
|
|
75878
77645
|
load() {
|
|
75879
77646
|
const out = /* @__PURE__ */ new Map();
|
|
75880
|
-
if (!
|
|
77647
|
+
if (!existsSync30(this.path)) {
|
|
75881
77648
|
const dir = dirname17(this.path);
|
|
75882
|
-
if (!
|
|
77649
|
+
if (!existsSync30(dir)) mkdirSync12(dir, { recursive: true, mode: 448 });
|
|
75883
77650
|
return out;
|
|
75884
77651
|
}
|
|
75885
|
-
const raw =
|
|
77652
|
+
const raw = readFileSync26(this.path);
|
|
75886
77653
|
const json = this.decodeFile(raw);
|
|
75887
77654
|
for (const [sourceId, b64] of Object.entries(json)) {
|
|
75888
77655
|
try {
|
|
@@ -75991,6 +77758,7 @@ export {
|
|
|
75991
77758
|
DEFAULT_TYPE_ALIASES,
|
|
75992
77759
|
EMBEDDINGS_TABLE,
|
|
75993
77760
|
EmbeddingDimensionMismatchError,
|
|
77761
|
+
EmbeddingScanTooLargeError,
|
|
75994
77762
|
FileSourceKeyStore,
|
|
75995
77763
|
FoldCache,
|
|
75996
77764
|
InMemorySourceKeyStore,
|
|
@@ -76054,6 +77822,7 @@ export {
|
|
|
76054
77822
|
createS3Store,
|
|
76055
77823
|
createSQLiteStateStore,
|
|
76056
77824
|
decrypt,
|
|
77825
|
+
dedupeAndDetectViews,
|
|
76057
77826
|
defaultWorkspaceYaml,
|
|
76058
77827
|
deleteDbCredential,
|
|
76059
77828
|
deleteToken,
|
|
@@ -76061,6 +77830,9 @@ export {
|
|
|
76061
77830
|
deriveKey,
|
|
76062
77831
|
describeImage,
|
|
76063
77832
|
describePdf,
|
|
77833
|
+
detectAsOf,
|
|
77834
|
+
detectAsOfCandidates,
|
|
77835
|
+
detectAsOfColumns,
|
|
76064
77836
|
detectRetrievalRegressions,
|
|
76065
77837
|
diagnoseRetrieval,
|
|
76066
77838
|
discoverCloudTables,
|
|
@@ -76078,6 +77850,7 @@ export {
|
|
|
76078
77850
|
entityFileNames,
|
|
76079
77851
|
estimateTokens,
|
|
76080
77852
|
evaluateRetrieval,
|
|
77853
|
+
excelToRecords,
|
|
76081
77854
|
extractEdgesFromColumn,
|
|
76082
77855
|
extractObjects,
|
|
76083
77856
|
filePresignSql,
|
|
@@ -76106,6 +77879,8 @@ export {
|
|
|
76106
77879
|
hashFile,
|
|
76107
77880
|
hybridSearch,
|
|
76108
77881
|
importLegacyUserConfig,
|
|
77882
|
+
inferFieldType,
|
|
77883
|
+
inferSchema,
|
|
76109
77884
|
installCloudRls,
|
|
76110
77885
|
installCloudSettings,
|
|
76111
77886
|
installFilePresigner,
|
|
@@ -76124,15 +77899,19 @@ export {
|
|
|
76124
77899
|
loadColumnPolicy,
|
|
76125
77900
|
manifestPath,
|
|
76126
77901
|
markdownTable,
|
|
77902
|
+
matchSchemaToExisting,
|
|
77903
|
+
materializeImport,
|
|
76127
77904
|
memberGroupFor,
|
|
76128
77905
|
memberRoleName,
|
|
76129
77906
|
migrateLatticeData,
|
|
76130
77907
|
neighbors,
|
|
77908
|
+
normalizeName,
|
|
76131
77909
|
observationVisible,
|
|
76132
77910
|
observationsFromChange,
|
|
76133
77911
|
openTargetLatticeForMigration,
|
|
76134
77912
|
openUnderSource,
|
|
76135
77913
|
organizeSource,
|
|
77914
|
+
parseCellDate,
|
|
76136
77915
|
parseConfigFile,
|
|
76137
77916
|
parseConfigString,
|
|
76138
77917
|
parseMarkdownEntries,
|
|
@@ -76160,6 +77939,7 @@ export {
|
|
|
76160
77939
|
registryPath,
|
|
76161
77940
|
removeEdge,
|
|
76162
77941
|
removeEmbedding,
|
|
77942
|
+
renameEntities,
|
|
76163
77943
|
resolveActiveS3Config,
|
|
76164
77944
|
resolveLatticeRoot,
|
|
76165
77945
|
resolveProvenanceFields,
|
|
@@ -76190,6 +77970,7 @@ export {
|
|
|
76190
77970
|
setTableNeverShare,
|
|
76191
77971
|
shredSource,
|
|
76192
77972
|
slugify,
|
|
77973
|
+
sourceRecords,
|
|
76193
77974
|
startGuiServer,
|
|
76194
77975
|
storeEmbedding,
|
|
76195
77976
|
summarizeText,
|