latticesql 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/cli.js +1899 -135
- package/dist/index.cjs +1948 -153
- package/dist/index.d.cts +360 -1
- package/dist/index.d.ts +360 -1
- package/dist/index.js +1908 -127
- package/docs/api-reference.md +60 -4
- package/docs/architecture.md +24 -0
- package/docs/assistant.md +23 -0
- package/docs/examples/dashboard.html +284 -0
- package/docs/importing.md +118 -0
- package/docs/retrieval.md +31 -0
- package/package.json +7 -3
package/dist/index.cjs
CHANGED
|
@@ -5536,7 +5536,9 @@ function withCredentialLock(fn) {
|
|
|
5536
5536
|
fd = (0, import_node_fs9.openSync)(lockPath, "wx");
|
|
5537
5537
|
break;
|
|
5538
5538
|
} catch (err) {
|
|
5539
|
-
|
|
5539
|
+
const code = err.code;
|
|
5540
|
+
const contended = code === "EEXIST" || process.platform === "win32" && (code === "EPERM" || code === "EACCES");
|
|
5541
|
+
if (!contended) throw err;
|
|
5540
5542
|
try {
|
|
5541
5543
|
if (Date.now() - (0, import_node_fs9.statSync)(lockPath).mtimeMs > LOCK_STALE_MS) {
|
|
5542
5544
|
(0, import_node_fs9.unlinkSync)(lockPath);
|
|
@@ -6428,6 +6430,7 @@ function deriveCanonicalContexts(tables) {
|
|
|
6428
6430
|
childrenOf.set(rel.table, list);
|
|
6429
6431
|
}
|
|
6430
6432
|
}
|
|
6433
|
+
const byName = new Map(tables.map((t8) => [t8.name, t8.definition]));
|
|
6431
6434
|
const out = [];
|
|
6432
6435
|
for (const { name, definition } of tables) {
|
|
6433
6436
|
const files = {};
|
|
@@ -6443,11 +6446,32 @@ function deriveCanonicalContexts(tables) {
|
|
|
6443
6446
|
};
|
|
6444
6447
|
}
|
|
6445
6448
|
for (const child of childrenOf.get(name) ?? []) {
|
|
6446
|
-
|
|
6447
|
-
|
|
6448
|
-
|
|
6449
|
-
|
|
6450
|
-
|
|
6449
|
+
const childDef = byName.get(child.table);
|
|
6450
|
+
const childBt = childDef ? belongsToRelations(childDef) : [];
|
|
6451
|
+
const [rel0, rel1] = childBt;
|
|
6452
|
+
if (childDef && rel0 && rel1 && isRenderJunction(childDef, childBt)) {
|
|
6453
|
+
const localRel = rel0.foreignKey === child.foreignKey ? rel0 : rel1;
|
|
6454
|
+
const remoteRel = localRel === rel0 ? rel1 : rel0;
|
|
6455
|
+
const fileKey = remoteRel.table === name ? `${child.table.toUpperCase()}__${remoteRel.foreignKey.toUpperCase()}.md` : `${remoteRel.table.toUpperCase()}.md`;
|
|
6456
|
+
files[fileKey] = {
|
|
6457
|
+
source: {
|
|
6458
|
+
type: "manyToMany",
|
|
6459
|
+
junctionTable: child.table,
|
|
6460
|
+
localKey: localRel.foreignKey,
|
|
6461
|
+
remoteKey: remoteRel.foreignKey,
|
|
6462
|
+
remoteTable: remoteRel.table,
|
|
6463
|
+
references: remoteRel.references ?? "id"
|
|
6464
|
+
},
|
|
6465
|
+
render: renderRelated(remoteRel.table),
|
|
6466
|
+
omitIfEmpty: true
|
|
6467
|
+
};
|
|
6468
|
+
} else {
|
|
6469
|
+
files[`${child.table.toUpperCase()}.md`] = {
|
|
6470
|
+
source: { type: "hasMany", table: child.table, foreignKey: child.foreignKey },
|
|
6471
|
+
render: renderRelated(child.table),
|
|
6472
|
+
omitIfEmpty: true
|
|
6473
|
+
};
|
|
6474
|
+
}
|
|
6451
6475
|
}
|
|
6452
6476
|
out.push({
|
|
6453
6477
|
table: name,
|
|
@@ -6460,6 +6484,15 @@ function deriveCanonicalContexts(tables) {
|
|
|
6460
6484
|
}
|
|
6461
6485
|
return out;
|
|
6462
6486
|
}
|
|
6487
|
+
function isRenderJunction(def, bt) {
|
|
6488
|
+
if (bt.length !== 2) return false;
|
|
6489
|
+
const fks = new Set(bt.map((r6) => r6.foreignKey));
|
|
6490
|
+
if (fks.size !== 2) return false;
|
|
6491
|
+
const pk = Array.isArray(def.primaryKey) ? def.primaryKey : def.primaryKey != null ? [def.primaryKey] : [];
|
|
6492
|
+
if (pk.length === 2 && pk.every((c6) => fks.has(c6))) return true;
|
|
6493
|
+
const SYSTEM2 = /* @__PURE__ */ new Set(["id", "created_at", "updated_at", "deleted_at"]);
|
|
6494
|
+
return Object.keys(def.columns).every((c6) => fks.has(c6) || SYSTEM2.has(c6));
|
|
6495
|
+
}
|
|
6463
6496
|
function belongsToRelations(def) {
|
|
6464
6497
|
return Object.values(def.relations ?? {}).filter(
|
|
6465
6498
|
(r6) => r6.type === "belongsTo"
|
|
@@ -6911,6 +6944,19 @@ var init_vector_index = __esm({
|
|
|
6911
6944
|
}
|
|
6912
6945
|
});
|
|
6913
6946
|
|
|
6947
|
+
// src/search/limits.ts
|
|
6948
|
+
function clampTopK(topK) {
|
|
6949
|
+
if (!Number.isFinite(topK)) return 1;
|
|
6950
|
+
return Math.min(Math.max(1, Math.floor(topK)), SEARCH_TOPK_MAX);
|
|
6951
|
+
}
|
|
6952
|
+
var SEARCH_TOPK_MAX;
|
|
6953
|
+
var init_limits = __esm({
|
|
6954
|
+
"src/search/limits.ts"() {
|
|
6955
|
+
"use strict";
|
|
6956
|
+
SEARCH_TOPK_MAX = 1e3;
|
|
6957
|
+
}
|
|
6958
|
+
});
|
|
6959
|
+
|
|
6914
6960
|
// src/search/embeddings.ts
|
|
6915
6961
|
async function ensureEmbeddingsTable(adapter) {
|
|
6916
6962
|
let cols = [];
|
|
@@ -7057,9 +7103,10 @@ function cosineSimilarity(a6, b6) {
|
|
|
7057
7103
|
}
|
|
7058
7104
|
async function searchByEmbedding(adapter, table, queryText, config, topK, minScore, pkColumn = "id") {
|
|
7059
7105
|
const queryVector = await config.embed(queryText);
|
|
7106
|
+
const k6 = clampTopK(topK);
|
|
7060
7107
|
let ranked;
|
|
7061
7108
|
if (await vectorIndexAvailable(adapter) && await hasVectorIndex(adapter, table)) {
|
|
7062
|
-
const hits = await searchVectorIndex(adapter, table, queryVector,
|
|
7109
|
+
const hits = await searchVectorIndex(adapter, table, queryVector, k6 * 4, minScore);
|
|
7063
7110
|
ranked = hits.map((h6) => ({
|
|
7064
7111
|
pk: h6.pk,
|
|
7065
7112
|
score: h6.score,
|
|
@@ -7067,7 +7114,7 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
7067
7114
|
content: h6.content
|
|
7068
7115
|
}));
|
|
7069
7116
|
} else {
|
|
7070
|
-
ranked = await scanChunks(adapter, table, queryVector, minScore);
|
|
7117
|
+
ranked = await scanChunks(adapter, table, queryVector, minScore, config.maxScanChunks);
|
|
7071
7118
|
}
|
|
7072
7119
|
const bestByRow = /* @__PURE__ */ new Map();
|
|
7073
7120
|
for (const r6 of ranked) {
|
|
@@ -7092,11 +7139,20 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
7092
7139
|
if (r6.content !== null) result.matchedContent = r6.content;
|
|
7093
7140
|
}
|
|
7094
7141
|
results.push(result);
|
|
7095
|
-
if (results.length >=
|
|
7142
|
+
if (results.length >= k6) break;
|
|
7096
7143
|
}
|
|
7097
7144
|
return results;
|
|
7098
7145
|
}
|
|
7099
|
-
async function scanChunks(adapter, table, queryVector, minScore) {
|
|
7146
|
+
async function scanChunks(adapter, table, queryVector, minScore, maxScanChunks) {
|
|
7147
|
+
if (maxScanChunks !== void 0) {
|
|
7148
|
+
const countRows = await allAsyncOrSync(
|
|
7149
|
+
adapter,
|
|
7150
|
+
`SELECT COUNT(*) AS n FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
7151
|
+
[table]
|
|
7152
|
+
);
|
|
7153
|
+
const n3 = Number(countRows[0]?.n ?? 0);
|
|
7154
|
+
if (n3 > maxScanChunks) throw new EmbeddingScanTooLargeError(table, n3, maxScanChunks);
|
|
7155
|
+
}
|
|
7100
7156
|
const stored = await allAsyncOrSync(
|
|
7101
7157
|
adapter,
|
|
7102
7158
|
`SELECT "row_pk", "chunk_index", "content", "embedding", "vec_dim" FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
@@ -7206,13 +7262,14 @@ async function refreshEmbeddings(adapter, table, config, pkColumn = "id", opts =
|
|
|
7206
7262
|
}
|
|
7207
7263
|
return { embedded, skipped, removed };
|
|
7208
7264
|
}
|
|
7209
|
-
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError;
|
|
7265
|
+
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError, EmbeddingScanTooLargeError;
|
|
7210
7266
|
var init_embeddings = __esm({
|
|
7211
7267
|
"src/search/embeddings.ts"() {
|
|
7212
7268
|
"use strict";
|
|
7213
7269
|
init_adapter();
|
|
7214
7270
|
init_chunking();
|
|
7215
7271
|
init_vector_index();
|
|
7272
|
+
init_limits();
|
|
7216
7273
|
EMBEDDINGS_TABLE = "_lattice_embeddings";
|
|
7217
7274
|
EmbeddingDimensionMismatchError = class extends Error {
|
|
7218
7275
|
constructor(table, expected, found) {
|
|
@@ -7225,6 +7282,17 @@ var init_embeddings = __esm({
|
|
|
7225
7282
|
this.name = "EmbeddingDimensionMismatchError";
|
|
7226
7283
|
}
|
|
7227
7284
|
};
|
|
7285
|
+
EmbeddingScanTooLargeError = class extends Error {
|
|
7286
|
+
constructor(table, found, limit) {
|
|
7287
|
+
super(
|
|
7288
|
+
`Embedding scan on "${table}" would read ${String(found)} stored chunk vectors, over the configured maxScanChunks of ${String(limit)}. Add a native vector index (pgvector) for this table or raise maxScanChunks \u2014 Lattice will not silently truncate the scan, which would return incomplete results.`
|
|
7289
|
+
);
|
|
7290
|
+
this.table = table;
|
|
7291
|
+
this.found = found;
|
|
7292
|
+
this.limit = limit;
|
|
7293
|
+
this.name = "EmbeddingScanTooLargeError";
|
|
7294
|
+
}
|
|
7295
|
+
};
|
|
7228
7296
|
}
|
|
7229
7297
|
});
|
|
7230
7298
|
|
|
@@ -7577,7 +7645,7 @@ async function fetchLiveRows2(adapter, table, ids, pkColumn) {
|
|
|
7577
7645
|
return out;
|
|
7578
7646
|
}
|
|
7579
7647
|
async function hybridSearch(adapter, table, query, opts = {}) {
|
|
7580
|
-
const topK = opts.topK ?? 10;
|
|
7648
|
+
const topK = clampTopK(opts.topK ?? 10);
|
|
7581
7649
|
const rrfK = opts.rrfK ?? 60;
|
|
7582
7650
|
const pool = opts.poolSize ?? Math.max(topK * 4, 20);
|
|
7583
7651
|
const pkColumn = opts.pkColumn ?? "id";
|
|
@@ -7678,6 +7746,7 @@ var init_hybrid = __esm({
|
|
|
7678
7746
|
init_fts();
|
|
7679
7747
|
init_ranking();
|
|
7680
7748
|
init_rerank();
|
|
7749
|
+
init_limits();
|
|
7681
7750
|
}
|
|
7682
7751
|
});
|
|
7683
7752
|
|
|
@@ -8405,6 +8474,26 @@ RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
|
8405
8474
|
);
|
|
8406
8475
|
$fn$;
|
|
8407
8476
|
|
|
8477
|
+
-- Delete-event visibility, decided from the PRE-DELETE snapshot the delete trigger
|
|
8478
|
+
-- captures (the live row + its ownership record are gone after a delete, so
|
|
8479
|
+
-- lattice_row_visible can't be used). Keyed on session_user, SECURITY DEFINER \u2014
|
|
8480
|
+
-- the same per-recipient gate. MUST MIRROR lattice_row_visible's rule: the row is
|
|
8481
|
+
-- visible iff this member owned it, OR it was 'everyone', OR it was 'custom' and
|
|
8482
|
+
-- this member was a grantee. A NULL owner snapshot (a legacy delete emitted before
|
|
8483
|
+
-- the snapshot columns, or a row with no ownership record) yields false \u2014 fail
|
|
8484
|
+
-- closed, never forward. (tests/integration assert this agrees with
|
|
8485
|
+
-- lattice_row_visible for all three visibility states \u2014 the no-drift guard.)
|
|
8486
|
+
CREATE OR REPLACE FUNCTION lattice_delete_visible(
|
|
8487
|
+
p_owner_role text, p_visibility text, p_grantees text[]
|
|
8488
|
+
)
|
|
8489
|
+
RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
8490
|
+
SELECT p_owner_role IS NOT NULL AND (
|
|
8491
|
+
p_owner_role = session_user
|
|
8492
|
+
OR p_visibility = 'everyone'
|
|
8493
|
+
OR (p_visibility = 'custom' AND session_user = ANY(COALESCE(p_grantees, ARRAY[]::text[])))
|
|
8494
|
+
);
|
|
8495
|
+
$fn$;
|
|
8496
|
+
|
|
8408
8497
|
-- Shared owner gate: raises unless the connected member owns (p_table, p_pk).
|
|
8409
8498
|
-- p_action is spliced into the message so every caller keeps its exact wording.
|
|
8410
8499
|
-- SECURITY DEFINER + session_user (never current_user), the cloud identity invariant.
|
|
@@ -8579,6 +8668,14 @@ CREATE TABLE IF NOT EXISTS "__lattice_changes" (
|
|
|
8579
8668
|
"created_at" timestamptz NOT NULL DEFAULT now()
|
|
8580
8669
|
);
|
|
8581
8670
|
|
|
8671
|
+
-- Pre-delete visibility snapshot columns (added to existing clouds via ADD COLUMN
|
|
8672
|
+
-- IF NOT EXISTS). A delete event carries the row's visibility AT DELETE TIME so the
|
|
8673
|
+
-- live fan-out can gate it per recipient even though the ownership record is gone.
|
|
8674
|
+
-- NULL on upserts.
|
|
8675
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_owner_role" text;
|
|
8676
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_visibility" text;
|
|
8677
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_grantees" text[];
|
|
8678
|
+
|
|
8582
8679
|
CREATE OR REPLACE FUNCTION lattice_notify_change() RETURNS trigger
|
|
8583
8680
|
LANGUAGE plpgsql AS $fn$
|
|
8584
8681
|
BEGIN
|
|
@@ -8588,7 +8685,10 @@ BEGIN
|
|
|
8588
8685
|
'pk', NEW."pk",
|
|
8589
8686
|
'op', NEW."op",
|
|
8590
8687
|
'owner_role', NEW."owner_role",
|
|
8591
|
-
'created_at', NEW."created_at"
|
|
8688
|
+
'created_at', NEW."created_at",
|
|
8689
|
+
'del_owner_role', NEW."del_owner_role",
|
|
8690
|
+
'del_visibility', NEW."del_visibility",
|
|
8691
|
+
'del_grantees', NEW."del_grantees"
|
|
8592
8692
|
)::text);
|
|
8593
8693
|
RETURN NEW;
|
|
8594
8694
|
END $fn$;
|
|
@@ -8794,10 +8894,22 @@ BEGIN
|
|
|
8794
8894
|
VALUES (${lit}, ${pkNew}, 'upsert', session_user);
|
|
8795
8895
|
RETURN NEW;
|
|
8796
8896
|
ELSIF TG_OP = 'DELETE' THEN
|
|
8897
|
+
-- Snapshot the row's visibility BEFORE the cascade removes its ownership +
|
|
8898
|
+
-- grant records, so the realtime fan-out can gate the delete event per
|
|
8899
|
+
-- recipient (the live predicate can't \u2014 these records are gone post-delete).
|
|
8900
|
+
-- The grantee list is captured here because the grant rows are deleted in the
|
|
8901
|
+
-- same statement below; after that the 'custom' audience is unrecoverable.
|
|
8902
|
+
INSERT INTO "__lattice_changes"
|
|
8903
|
+
("table_name","pk","op","owner_role","del_owner_role","del_visibility","del_grantees")
|
|
8904
|
+
VALUES (${lit}, ${pkOld}, 'delete', session_user,
|
|
8905
|
+
(SELECT o."owner_role" FROM "__lattice_owners" o
|
|
8906
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
8907
|
+
(SELECT o."visibility" FROM "__lattice_owners" o
|
|
8908
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
8909
|
+
COALESCE((SELECT array_agg(g."grantee_role") FROM "__lattice_row_grants" g
|
|
8910
|
+
WHERE g."table_name" = ${lit} AND g."pk" = ${pkOld}), ARRAY[]::text[]));
|
|
8797
8911
|
DELETE FROM "__lattice_owners" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8798
8912
|
DELETE FROM "__lattice_row_grants" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8799
|
-
INSERT INTO "__lattice_changes" ("table_name","pk","op","owner_role")
|
|
8800
|
-
VALUES (${lit}, ${pkOld}, 'delete', session_user);
|
|
8801
8913
|
RETURN OLD;
|
|
8802
8914
|
END IF;
|
|
8803
8915
|
RETURN NEW;
|
|
@@ -13402,7 +13514,7 @@ var init_sleep = __esm({
|
|
|
13402
13514
|
"node_modules/@smithy/core/dist-es/submodules/client/util-waiter/utils/sleep.js"() {
|
|
13403
13515
|
"use strict";
|
|
13404
13516
|
sleep = (seconds) => {
|
|
13405
|
-
return new Promise((
|
|
13517
|
+
return new Promise((resolve17) => setTimeout(resolve17, seconds * 1e3));
|
|
13406
13518
|
};
|
|
13407
13519
|
}
|
|
13408
13520
|
});
|
|
@@ -13571,8 +13683,8 @@ var init_createWaiter = __esm({
|
|
|
13571
13683
|
init_waiter2();
|
|
13572
13684
|
abortTimeout = (abortSignal) => {
|
|
13573
13685
|
let onAbort;
|
|
13574
|
-
const promise = new Promise((
|
|
13575
|
-
onAbort = () =>
|
|
13686
|
+
const promise = new Promise((resolve17) => {
|
|
13687
|
+
onAbort = () => resolve17({ state: WaiterState.ABORTED });
|
|
13576
13688
|
if (typeof abortSignal.addEventListener === "function") {
|
|
13577
13689
|
abortSignal.addEventListener("abort", onAbort);
|
|
13578
13690
|
} else {
|
|
@@ -16454,7 +16566,7 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
16454
16566
|
};
|
|
16455
16567
|
imdsHttpGet = async ({ hostname, path: path2 }) => {
|
|
16456
16568
|
const { request } = await import("http");
|
|
16457
|
-
return new Promise((
|
|
16569
|
+
return new Promise((resolve17, reject) => {
|
|
16458
16570
|
const req = request({
|
|
16459
16571
|
method: "GET",
|
|
16460
16572
|
hostname: hostname.replace(/^\[(.+)]$/, "$1"),
|
|
@@ -16480,7 +16592,7 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
16480
16592
|
const chunks = [];
|
|
16481
16593
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
16482
16594
|
res.on("end", () => {
|
|
16483
|
-
|
|
16595
|
+
resolve17(Buffer.concat(chunks));
|
|
16484
16596
|
req.destroy();
|
|
16485
16597
|
});
|
|
16486
16598
|
});
|
|
@@ -18242,7 +18354,7 @@ async function collectStream(stream) {
|
|
|
18242
18354
|
return collected;
|
|
18243
18355
|
}
|
|
18244
18356
|
function readToBase64(blob) {
|
|
18245
|
-
return new Promise((
|
|
18357
|
+
return new Promise((resolve17, reject) => {
|
|
18246
18358
|
const reader = new FileReader();
|
|
18247
18359
|
reader.onloadend = () => {
|
|
18248
18360
|
if (reader.readyState !== 2) {
|
|
@@ -18251,7 +18363,7 @@ function readToBase64(blob) {
|
|
|
18251
18363
|
const result = reader.result ?? "";
|
|
18252
18364
|
const commaIndex = result.indexOf(",");
|
|
18253
18365
|
const dataOffset = commaIndex > -1 ? commaIndex + 1 : result.length;
|
|
18254
|
-
|
|
18366
|
+
resolve17(result.substring(dataOffset));
|
|
18255
18367
|
};
|
|
18256
18368
|
reader.onabort = () => reject(new Error("Read aborted"));
|
|
18257
18369
|
reader.onerror = () => reject(reader.error);
|
|
@@ -18379,7 +18491,7 @@ var init_stream_collector = __esm({
|
|
|
18379
18491
|
if (isReadableStreamInstance(stream)) {
|
|
18380
18492
|
return collectReadableStream(stream);
|
|
18381
18493
|
}
|
|
18382
|
-
return new Promise((
|
|
18494
|
+
return new Promise((resolve17, reject) => {
|
|
18383
18495
|
const collector = new Collector();
|
|
18384
18496
|
stream.pipe(collector);
|
|
18385
18497
|
stream.on("error", (err) => {
|
|
@@ -18389,7 +18501,7 @@ var init_stream_collector = __esm({
|
|
|
18389
18501
|
collector.on("error", reject);
|
|
18390
18502
|
collector.on("finish", function() {
|
|
18391
18503
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
18392
|
-
|
|
18504
|
+
resolve17(bytes);
|
|
18393
18505
|
});
|
|
18394
18506
|
});
|
|
18395
18507
|
};
|
|
@@ -18536,11 +18648,11 @@ var init_SerdeContext = __esm({
|
|
|
18536
18648
|
// node_modules/tslib/tslib.es6.mjs
|
|
18537
18649
|
function __awaiter(thisArg, _arguments, P2, generator) {
|
|
18538
18650
|
function adopt(value) {
|
|
18539
|
-
return value instanceof P2 ? value : new P2(function(
|
|
18540
|
-
|
|
18651
|
+
return value instanceof P2 ? value : new P2(function(resolve17) {
|
|
18652
|
+
resolve17(value);
|
|
18541
18653
|
});
|
|
18542
18654
|
}
|
|
18543
|
-
return new (P2 || (P2 = Promise))(function(
|
|
18655
|
+
return new (P2 || (P2 = Promise))(function(resolve17, reject) {
|
|
18544
18656
|
function fulfilled(value) {
|
|
18545
18657
|
try {
|
|
18546
18658
|
step(generator.next(value));
|
|
@@ -18556,7 +18668,7 @@ function __awaiter(thisArg, _arguments, P2, generator) {
|
|
|
18556
18668
|
}
|
|
18557
18669
|
}
|
|
18558
18670
|
function step(result) {
|
|
18559
|
-
result.done ?
|
|
18671
|
+
result.done ? resolve17(result.value) : adopt(result.value).then(fulfilled, rejected);
|
|
18560
18672
|
}
|
|
18561
18673
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
18562
18674
|
});
|
|
@@ -19752,7 +19864,7 @@ async function* readableToIterable(readStream) {
|
|
|
19752
19864
|
streamEnded = true;
|
|
19753
19865
|
});
|
|
19754
19866
|
while (!generationEnded) {
|
|
19755
|
-
const value = await new Promise((
|
|
19867
|
+
const value = await new Promise((resolve17) => setTimeout(() => resolve17(records.shift()), 0));
|
|
19756
19868
|
if (value) {
|
|
19757
19869
|
yield value;
|
|
19758
19870
|
}
|
|
@@ -21298,7 +21410,7 @@ var init_retryMiddleware = __esm({
|
|
|
21298
21410
|
init_constants5();
|
|
21299
21411
|
init_parseRetryAfterHeader();
|
|
21300
21412
|
init_util2();
|
|
21301
|
-
cooldown = (ms) => new Promise((
|
|
21413
|
+
cooldown = (ms) => new Promise((resolve17) => setTimeout(resolve17, ms));
|
|
21302
21414
|
isRetryStrategyV2 = (retryStrategy) => typeof retryStrategy.acquireInitialRetryToken !== "undefined" && typeof retryStrategy.refreshRetryTokenForRetry !== "undefined" && typeof retryStrategy.recordSuccess !== "undefined";
|
|
21303
21415
|
getRetryErrorInfo = (error, logger2) => {
|
|
21304
21416
|
const errorInfo = {
|
|
@@ -21397,7 +21509,7 @@ var init_DefaultRateLimiter = __esm({
|
|
|
21397
21509
|
this.refillTokenBucket();
|
|
21398
21510
|
while (amount > this.availableTokens) {
|
|
21399
21511
|
const delay = (amount - this.availableTokens) / this.fillRate * 1e3;
|
|
21400
|
-
await new Promise((
|
|
21512
|
+
await new Promise((resolve17) => _DefaultRateLimiter.setTimeoutFn(resolve17, delay));
|
|
21401
21513
|
this.refillTokenBucket();
|
|
21402
21514
|
}
|
|
21403
21515
|
this.availableTokens = this.availableTokens - amount;
|
|
@@ -25067,8 +25179,8 @@ var init_SignatureV4 = __esm({
|
|
|
25067
25179
|
priorSignature: signableMessage.priorSignature,
|
|
25068
25180
|
eventStreamCredentials
|
|
25069
25181
|
});
|
|
25070
|
-
return promise.then((
|
|
25071
|
-
return { message: signableMessage.message, signature };
|
|
25182
|
+
return promise.then((signature2) => {
|
|
25183
|
+
return { message: signableMessage.message, signature: signature2 };
|
|
25072
25184
|
});
|
|
25073
25185
|
}
|
|
25074
25186
|
async signString(stringToSign, { signingDate = /* @__PURE__ */ new Date(), signingRegion, signingService, eventStreamCredentials } = {}) {
|
|
@@ -25096,8 +25208,8 @@ var init_SignatureV4 = __esm({
|
|
|
25096
25208
|
request.headers[SHA256_HEADER] = payloadHash;
|
|
25097
25209
|
}
|
|
25098
25210
|
const canonicalHeaders = getCanonicalHeaders(request, unsignableHeaders, signableHeaders);
|
|
25099
|
-
const
|
|
25100
|
-
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${
|
|
25211
|
+
const signature2 = await this.getSignature(longDate, scope, this.getSigningKey(credentials, region, shortDate, signingService), this.createCanonicalRequest(request, canonicalHeaders, payloadHash));
|
|
25212
|
+
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${signature2}`;
|
|
25101
25213
|
return request;
|
|
25102
25214
|
}
|
|
25103
25215
|
async getSignature(longDate, credentialScope, keyPromise, canonicalRequest) {
|
|
@@ -40687,7 +40799,7 @@ var init_node_http = __esm({
|
|
|
40687
40799
|
|
|
40688
40800
|
// node_modules/@smithy/credential-provider-imds/dist-es/remoteProvider/httpRequest.js
|
|
40689
40801
|
function httpRequest(options) {
|
|
40690
|
-
return new Promise((
|
|
40802
|
+
return new Promise((resolve17, reject) => {
|
|
40691
40803
|
const req = import_node_http.default.request({
|
|
40692
40804
|
method: "GET",
|
|
40693
40805
|
...options,
|
|
@@ -40712,7 +40824,7 @@ function httpRequest(options) {
|
|
|
40712
40824
|
chunks.push(chunk);
|
|
40713
40825
|
});
|
|
40714
40826
|
res.on("end", () => {
|
|
40715
|
-
|
|
40827
|
+
resolve17(Buffer.concat(chunks));
|
|
40716
40828
|
req.destroy();
|
|
40717
40829
|
});
|
|
40718
40830
|
});
|
|
@@ -41357,21 +41469,21 @@ async function writeRequestBody(httpRequest2, request, maxContinueTimeoutMs = MI
|
|
|
41357
41469
|
let sendBody = true;
|
|
41358
41470
|
if (!externalAgent && expect === "100-continue") {
|
|
41359
41471
|
sendBody = await Promise.race([
|
|
41360
|
-
new Promise((
|
|
41361
|
-
timeoutId = Number(timing.setTimeout(() =>
|
|
41472
|
+
new Promise((resolve17) => {
|
|
41473
|
+
timeoutId = Number(timing.setTimeout(() => resolve17(true), Math.max(MIN_WAIT_TIME, maxContinueTimeoutMs)));
|
|
41362
41474
|
}),
|
|
41363
|
-
new Promise((
|
|
41475
|
+
new Promise((resolve17) => {
|
|
41364
41476
|
httpRequest2.on("continue", () => {
|
|
41365
41477
|
timing.clearTimeout(timeoutId);
|
|
41366
|
-
|
|
41478
|
+
resolve17(true);
|
|
41367
41479
|
});
|
|
41368
41480
|
httpRequest2.on("response", () => {
|
|
41369
41481
|
timing.clearTimeout(timeoutId);
|
|
41370
|
-
|
|
41482
|
+
resolve17(false);
|
|
41371
41483
|
});
|
|
41372
41484
|
httpRequest2.on("error", () => {
|
|
41373
41485
|
timing.clearTimeout(timeoutId);
|
|
41374
|
-
|
|
41486
|
+
resolve17(false);
|
|
41375
41487
|
});
|
|
41376
41488
|
})
|
|
41377
41489
|
]);
|
|
@@ -41470,13 +41582,13 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41470
41582
|
return socketWarningTimestamp;
|
|
41471
41583
|
}
|
|
41472
41584
|
constructor(options) {
|
|
41473
|
-
this.configProvider = new Promise((
|
|
41585
|
+
this.configProvider = new Promise((resolve17, reject) => {
|
|
41474
41586
|
if (typeof options === "function") {
|
|
41475
41587
|
options().then((_options) => {
|
|
41476
|
-
|
|
41588
|
+
resolve17(this.resolveDefaultConfig(_options));
|
|
41477
41589
|
}).catch(reject);
|
|
41478
41590
|
} else {
|
|
41479
|
-
|
|
41591
|
+
resolve17(this.resolveDefaultConfig(options));
|
|
41480
41592
|
}
|
|
41481
41593
|
});
|
|
41482
41594
|
}
|
|
@@ -41507,7 +41619,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41507
41619
|
timing.clearTimeout(socketTimeoutId);
|
|
41508
41620
|
timing.clearTimeout(keepAliveTimeoutId);
|
|
41509
41621
|
};
|
|
41510
|
-
const
|
|
41622
|
+
const resolve17 = async (arg) => {
|
|
41511
41623
|
await writeRequestBodyPromise;
|
|
41512
41624
|
clearTimeouts();
|
|
41513
41625
|
_resolve(arg);
|
|
@@ -41571,7 +41683,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41571
41683
|
headers: getTransformedHeaders(res.headers),
|
|
41572
41684
|
body: res
|
|
41573
41685
|
});
|
|
41574
|
-
|
|
41686
|
+
resolve17({ response: httpResponse });
|
|
41575
41687
|
});
|
|
41576
41688
|
req.on("error", (err) => {
|
|
41577
41689
|
if (NODEJS_TIMEOUT_ERROR_CODES2.includes(err.code)) {
|
|
@@ -41724,7 +41836,7 @@ var init_stream_collector2 = __esm({
|
|
|
41724
41836
|
if (isReadableStreamInstance2(stream)) {
|
|
41725
41837
|
return collectReadableStream2(stream);
|
|
41726
41838
|
}
|
|
41727
|
-
return new Promise((
|
|
41839
|
+
return new Promise((resolve17, reject) => {
|
|
41728
41840
|
const collector = new Collector2();
|
|
41729
41841
|
stream.pipe(collector);
|
|
41730
41842
|
stream.on("error", (err) => {
|
|
@@ -41734,7 +41846,7 @@ var init_stream_collector2 = __esm({
|
|
|
41734
41846
|
collector.on("error", reject);
|
|
41735
41847
|
collector.on("finish", function() {
|
|
41736
41848
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
41737
|
-
|
|
41849
|
+
resolve17(bytes);
|
|
41738
41850
|
});
|
|
41739
41851
|
});
|
|
41740
41852
|
};
|
|
@@ -41856,7 +41968,7 @@ var init_retry_wrapper = __esm({
|
|
|
41856
41968
|
try {
|
|
41857
41969
|
return await toRetry();
|
|
41858
41970
|
} catch (e6) {
|
|
41859
|
-
await new Promise((
|
|
41971
|
+
await new Promise((resolve17) => setTimeout(resolve17, delayMs));
|
|
41860
41972
|
}
|
|
41861
41973
|
}
|
|
41862
41974
|
return await toRetry();
|
|
@@ -47091,14 +47203,14 @@ var init_readableStreamHasher = __esm({
|
|
|
47091
47203
|
const hash = new hashCtor();
|
|
47092
47204
|
const hashCalculator = new HashCalculator(hash);
|
|
47093
47205
|
readableStream.pipe(hashCalculator);
|
|
47094
|
-
return new Promise((
|
|
47206
|
+
return new Promise((resolve17, reject) => {
|
|
47095
47207
|
readableStream.on("error", (err) => {
|
|
47096
47208
|
hashCalculator.end();
|
|
47097
47209
|
reject(err);
|
|
47098
47210
|
});
|
|
47099
47211
|
hashCalculator.on("error", reject);
|
|
47100
47212
|
hashCalculator.on("finish", () => {
|
|
47101
|
-
hash.digest().then(
|
|
47213
|
+
hash.digest().then(resolve17).catch(reject);
|
|
47102
47214
|
});
|
|
47103
47215
|
});
|
|
47104
47216
|
};
|
|
@@ -52618,7 +52730,7 @@ function parsePageParam(raw, kind) {
|
|
|
52618
52730
|
}
|
|
52619
52731
|
function readJson(req, opts = {}) {
|
|
52620
52732
|
const maxBytes = opts.maxBytes ?? DEFAULT_BODY_MAX_BYTES;
|
|
52621
|
-
return new Promise((
|
|
52733
|
+
return new Promise((resolve17, reject) => {
|
|
52622
52734
|
let raw = "";
|
|
52623
52735
|
let overflowed = false;
|
|
52624
52736
|
req.setEncoding("utf8");
|
|
@@ -52634,7 +52746,7 @@ function readJson(req, opts = {}) {
|
|
|
52634
52746
|
req.on("end", () => {
|
|
52635
52747
|
if (overflowed) return;
|
|
52636
52748
|
try {
|
|
52637
|
-
|
|
52749
|
+
resolve17(raw ? JSON.parse(raw) : {});
|
|
52638
52750
|
} catch {
|
|
52639
52751
|
reject(new Error("Invalid JSON body"));
|
|
52640
52752
|
}
|
|
@@ -52654,11 +52766,12 @@ ${err.stack ?? ""}`);
|
|
|
52654
52766
|
sendJson(res, { error: err.message }, status);
|
|
52655
52767
|
}
|
|
52656
52768
|
}
|
|
52657
|
-
var DEFAULT_BODY_MAX_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
52769
|
+
var DEFAULT_BODY_MAX_BYTES, MAX_INGEST_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
52658
52770
|
var init_http2 = __esm({
|
|
52659
52771
|
"src/gui/http.ts"() {
|
|
52660
52772
|
"use strict";
|
|
52661
52773
|
DEFAULT_BODY_MAX_BYTES = 1e6;
|
|
52774
|
+
MAX_INGEST_BYTES = 5e7;
|
|
52662
52775
|
MAX_ROWS_PAGE = 1e3;
|
|
52663
52776
|
DEFAULT_ROWS_PAGE = 500;
|
|
52664
52777
|
BodyTooLargeError = class extends Error {
|
|
@@ -53651,7 +53764,7 @@ var init_oauth = __esm({
|
|
|
53651
53764
|
|
|
53652
53765
|
// src/gui/assistant-routes.ts
|
|
53653
53766
|
function readBuffer(req, maxBytes = 25e6) {
|
|
53654
|
-
return new Promise((
|
|
53767
|
+
return new Promise((resolve17, reject) => {
|
|
53655
53768
|
const chunks = [];
|
|
53656
53769
|
let size = 0;
|
|
53657
53770
|
req.on("data", (c6) => {
|
|
@@ -53660,7 +53773,7 @@ function readBuffer(req, maxBytes = 25e6) {
|
|
|
53660
53773
|
else chunks.push(c6);
|
|
53661
53774
|
});
|
|
53662
53775
|
req.on("end", () => {
|
|
53663
|
-
|
|
53776
|
+
resolve17(Buffer.concat(chunks));
|
|
53664
53777
|
});
|
|
53665
53778
|
req.on("error", reject);
|
|
53666
53779
|
});
|
|
@@ -54963,7 +55076,7 @@ async function takeHostSlot(host, minIntervalMs = urlIngestConfig().hostMinInter
|
|
|
54963
55076
|
const earliest = Math.max(now2, hostNextAllowed.get(key) ?? 0);
|
|
54964
55077
|
hostNextAllowed.set(key, earliest + minIntervalMs);
|
|
54965
55078
|
const wait = earliest - now2;
|
|
54966
|
-
if (wait > 0) await new Promise((
|
|
55079
|
+
if (wait > 0) await new Promise((resolve17) => setTimeout(resolve17, wait));
|
|
54967
55080
|
}
|
|
54968
55081
|
var Semaphore, FetchBudget, sharedGate, hostNextAllowed;
|
|
54969
55082
|
var init_fetch_policy = __esm({
|
|
@@ -54979,7 +55092,7 @@ var init_fetch_policy = __esm({
|
|
|
54979
55092
|
if (this.permits > 0) {
|
|
54980
55093
|
this.permits -= 1;
|
|
54981
55094
|
} else {
|
|
54982
|
-
await new Promise((
|
|
55095
|
+
await new Promise((resolve17) => this.waiters.push(resolve17));
|
|
54983
55096
|
}
|
|
54984
55097
|
let released = false;
|
|
54985
55098
|
return () => {
|
|
@@ -55215,8 +55328,8 @@ function fileContentGroups(rows, fuzzy, threshold) {
|
|
|
55215
55328
|
const t8 = get2(r6, "extracted_text");
|
|
55216
55329
|
return typeof t8 === "string" && t8.trim().length > 0;
|
|
55217
55330
|
}).map((r6) => {
|
|
55218
|
-
const
|
|
55219
|
-
const key = fuzzy ? "txt:" +
|
|
55331
|
+
const norm3 = normalizeText(get2(r6, "extracted_text"));
|
|
55332
|
+
const key = fuzzy ? "txt:" + norm3.slice(0, 2e3) : "txt:" + (0, import_node_crypto17.createHash)("sha256").update(norm3).digest("hex");
|
|
55220
55333
|
return { id: String(get2(r6, "id")), key, createdAt: cellStrOrNull(get2(r6, "created_at")) };
|
|
55221
55334
|
});
|
|
55222
55335
|
const txtGroups = findDuplicateGroups(txtItems, {
|
|
@@ -57583,6 +57696,7 @@ __export(index_exports, {
|
|
|
57583
57696
|
DEFAULT_TYPE_ALIASES: () => DEFAULT_TYPE_ALIASES,
|
|
57584
57697
|
EMBEDDINGS_TABLE: () => EMBEDDINGS_TABLE,
|
|
57585
57698
|
EmbeddingDimensionMismatchError: () => EmbeddingDimensionMismatchError,
|
|
57699
|
+
EmbeddingScanTooLargeError: () => EmbeddingScanTooLargeError,
|
|
57586
57700
|
FileSourceKeyStore: () => FileSourceKeyStore,
|
|
57587
57701
|
FoldCache: () => FoldCache,
|
|
57588
57702
|
InMemorySourceKeyStore: () => InMemorySourceKeyStore,
|
|
@@ -57646,6 +57760,7 @@ __export(index_exports, {
|
|
|
57646
57760
|
createS3Store: () => createS3Store,
|
|
57647
57761
|
createSQLiteStateStore: () => createSQLiteStateStore,
|
|
57648
57762
|
decrypt: () => decrypt,
|
|
57763
|
+
dedupeAndDetectViews: () => dedupeAndDetectViews,
|
|
57649
57764
|
defaultWorkspaceYaml: () => defaultWorkspaceYaml,
|
|
57650
57765
|
deleteDbCredential: () => deleteDbCredential,
|
|
57651
57766
|
deleteToken: () => deleteToken,
|
|
@@ -57653,6 +57768,9 @@ __export(index_exports, {
|
|
|
57653
57768
|
deriveKey: () => deriveKey,
|
|
57654
57769
|
describeImage: () => describeImage,
|
|
57655
57770
|
describePdf: () => describePdf,
|
|
57771
|
+
detectAsOf: () => detectAsOf,
|
|
57772
|
+
detectAsOfCandidates: () => detectAsOfCandidates,
|
|
57773
|
+
detectAsOfColumns: () => detectAsOfColumns,
|
|
57656
57774
|
detectRetrievalRegressions: () => detectRetrievalRegressions,
|
|
57657
57775
|
diagnoseRetrieval: () => diagnoseRetrieval,
|
|
57658
57776
|
discoverCloudTables: () => discoverCloudTables,
|
|
@@ -57670,6 +57788,7 @@ __export(index_exports, {
|
|
|
57670
57788
|
entityFileNames: () => entityFileNames,
|
|
57671
57789
|
estimateTokens: () => estimateTokens,
|
|
57672
57790
|
evaluateRetrieval: () => evaluateRetrieval,
|
|
57791
|
+
excelToRecords: () => excelToRecords,
|
|
57673
57792
|
extractEdgesFromColumn: () => extractEdgesFromColumn,
|
|
57674
57793
|
extractObjects: () => extractObjects,
|
|
57675
57794
|
filePresignSql: () => filePresignSql,
|
|
@@ -57698,6 +57817,8 @@ __export(index_exports, {
|
|
|
57698
57817
|
hashFile: () => hashFile,
|
|
57699
57818
|
hybridSearch: () => hybridSearch,
|
|
57700
57819
|
importLegacyUserConfig: () => importLegacyUserConfig,
|
|
57820
|
+
inferFieldType: () => inferFieldType,
|
|
57821
|
+
inferSchema: () => inferSchema,
|
|
57701
57822
|
installCloudRls: () => installCloudRls,
|
|
57702
57823
|
installCloudSettings: () => installCloudSettings,
|
|
57703
57824
|
installFilePresigner: () => installFilePresigner,
|
|
@@ -57716,15 +57837,19 @@ __export(index_exports, {
|
|
|
57716
57837
|
loadColumnPolicy: () => loadColumnPolicy,
|
|
57717
57838
|
manifestPath: () => manifestPath,
|
|
57718
57839
|
markdownTable: () => markdownTable,
|
|
57840
|
+
matchSchemaToExisting: () => matchSchemaToExisting,
|
|
57841
|
+
materializeImport: () => materializeImport,
|
|
57719
57842
|
memberGroupFor: () => memberGroupFor,
|
|
57720
57843
|
memberRoleName: () => memberRoleName,
|
|
57721
57844
|
migrateLatticeData: () => migrateLatticeData,
|
|
57722
57845
|
neighbors: () => neighbors,
|
|
57846
|
+
normalizeName: () => normalizeName,
|
|
57723
57847
|
observationVisible: () => observationVisible,
|
|
57724
57848
|
observationsFromChange: () => observationsFromChange,
|
|
57725
57849
|
openTargetLatticeForMigration: () => openTargetLatticeForMigration,
|
|
57726
57850
|
openUnderSource: () => openUnderSource,
|
|
57727
57851
|
organizeSource: () => organizeSource,
|
|
57852
|
+
parseCellDate: () => parseCellDate,
|
|
57728
57853
|
parseConfigFile: () => parseConfigFile,
|
|
57729
57854
|
parseConfigString: () => parseConfigString,
|
|
57730
57855
|
parseMarkdownEntries: () => parseMarkdownEntries,
|
|
@@ -57752,6 +57877,7 @@ __export(index_exports, {
|
|
|
57752
57877
|
registryPath: () => registryPath,
|
|
57753
57878
|
removeEdge: () => removeEdge,
|
|
57754
57879
|
removeEmbedding: () => removeEmbedding,
|
|
57880
|
+
renameEntities: () => renameEntities,
|
|
57755
57881
|
resolveActiveS3Config: () => resolveActiveS3Config,
|
|
57756
57882
|
resolveLatticeRoot: () => resolveLatticeRoot,
|
|
57757
57883
|
resolveProvenanceFields: () => resolveProvenanceFields,
|
|
@@ -57782,6 +57908,7 @@ __export(index_exports, {
|
|
|
57782
57908
|
setTableNeverShare: () => setTableNeverShare,
|
|
57783
57909
|
shredSource: () => shredSource,
|
|
57784
57910
|
slugify: () => slugify,
|
|
57911
|
+
sourceRecords: () => sourceRecords,
|
|
57785
57912
|
startGuiServer: () => startGuiServer,
|
|
57786
57913
|
storeEmbedding: () => storeEmbedding,
|
|
57787
57914
|
summarizeText: () => summarizeText,
|
|
@@ -58365,8 +58492,8 @@ function isRetryableDbError(err) {
|
|
|
58365
58492
|
return msg.includes("database is locked") || msg.includes("connection terminated") || msg.includes("connection reset") || msg.includes("server closed the connection");
|
|
58366
58493
|
}
|
|
58367
58494
|
var retryDepth = new import_node_async_hooks.AsyncLocalStorage();
|
|
58368
|
-
var defaultSleep = (ms) => new Promise((
|
|
58369
|
-
setTimeout(
|
|
58495
|
+
var defaultSleep = (ms) => new Promise((resolve17) => {
|
|
58496
|
+
setTimeout(resolve17, ms);
|
|
58370
58497
|
});
|
|
58371
58498
|
async function withRetry(fn, opts = {}) {
|
|
58372
58499
|
if (retryDepth.getStore()) return fn();
|
|
@@ -58565,13 +58692,13 @@ var import_node_crypto8 = require("crypto");
|
|
|
58565
58692
|
var import_node_fs14 = require("fs");
|
|
58566
58693
|
var import_node_path14 = require("path");
|
|
58567
58694
|
async function hashFile(srcPath) {
|
|
58568
|
-
return new Promise((
|
|
58695
|
+
return new Promise((resolve17, reject) => {
|
|
58569
58696
|
const hash = (0, import_node_crypto8.createHash)("sha256");
|
|
58570
58697
|
const stream = (0, import_node_fs14.createReadStream)(srcPath);
|
|
58571
58698
|
stream.on("data", (chunk) => hash.update(chunk));
|
|
58572
58699
|
stream.on("error", reject);
|
|
58573
58700
|
stream.on("end", () => {
|
|
58574
|
-
|
|
58701
|
+
resolve17(hash.digest("hex"));
|
|
58575
58702
|
});
|
|
58576
58703
|
});
|
|
58577
58704
|
}
|
|
@@ -59770,7 +59897,7 @@ init_summarize();
|
|
|
59770
59897
|
var import_node_http3 = require("http");
|
|
59771
59898
|
var import_node_child_process5 = require("child_process");
|
|
59772
59899
|
var import_ws = require("ws");
|
|
59773
|
-
var
|
|
59900
|
+
var import_node_path47 = require("path");
|
|
59774
59901
|
init_http2();
|
|
59775
59902
|
|
|
59776
59903
|
// src/gui/active-db.ts
|
|
@@ -59779,9 +59906,17 @@ init_members();
|
|
|
59779
59906
|
init_native_entities();
|
|
59780
59907
|
async function changeVisibleToActiveRole(db, payload) {
|
|
59781
59908
|
if (db.getDialect() !== "postgres") return true;
|
|
59782
|
-
if (payload.op === "delete" || payload.op === "DELETE") return true;
|
|
59783
59909
|
if (!payload.table_name || !payload.pk) return false;
|
|
59784
59910
|
try {
|
|
59911
|
+
if (isDeleteOp(payload.op)) {
|
|
59912
|
+
if (payload.del_owner_role == null) return false;
|
|
59913
|
+
const row2 = await getAsyncOrSync(
|
|
59914
|
+
db.adapter,
|
|
59915
|
+
`SELECT lattice_delete_visible(?, ?, ?::text[]) AS v`,
|
|
59916
|
+
[payload.del_owner_role, payload.del_visibility ?? null, payload.del_grantees ?? []]
|
|
59917
|
+
);
|
|
59918
|
+
return row2?.v === true || row2?.v === "t" || row2?.v === 1;
|
|
59919
|
+
}
|
|
59785
59920
|
const row = await getAsyncOrSync(db.adapter, `SELECT lattice_row_visible(?, ?) AS v`, [
|
|
59786
59921
|
payload.table_name,
|
|
59787
59922
|
payload.pk
|
|
@@ -60273,9 +60408,9 @@ var RealtimeBroker = class {
|
|
|
60273
60408
|
() => "ended"
|
|
60274
60409
|
// a graceful-close error is still "closed enough"
|
|
60275
60410
|
);
|
|
60276
|
-
const timedOut = new Promise((
|
|
60411
|
+
const timedOut = new Promise((resolve17) => {
|
|
60277
60412
|
timer = setTimeout(() => {
|
|
60278
|
-
|
|
60413
|
+
resolve17("timeout");
|
|
60279
60414
|
}, this.stopEndTimeoutMs);
|
|
60280
60415
|
timer.unref?.();
|
|
60281
60416
|
});
|
|
@@ -60320,7 +60455,10 @@ function parsePayload(raw) {
|
|
|
60320
60455
|
pk: typeof obj2.pk === "string" ? obj2.pk : null,
|
|
60321
60456
|
op: obj2.op,
|
|
60322
60457
|
owner_role: typeof obj2.owner_role === "string" ? obj2.owner_role : null,
|
|
60323
|
-
created_at: typeof obj2.created_at === "string" ? obj2.created_at : ""
|
|
60458
|
+
created_at: typeof obj2.created_at === "string" ? obj2.created_at : "",
|
|
60459
|
+
del_owner_role: typeof obj2.del_owner_role === "string" ? obj2.del_owner_role : null,
|
|
60460
|
+
del_visibility: typeof obj2.del_visibility === "string" ? obj2.del_visibility : null,
|
|
60461
|
+
del_grantees: Array.isArray(obj2.del_grantees) ? obj2.del_grantees.filter((g6) => typeof g6 === "string") : null
|
|
60324
60462
|
};
|
|
60325
60463
|
} catch {
|
|
60326
60464
|
return null;
|
|
@@ -61301,9 +61439,9 @@ function startBackgroundRender(active) {
|
|
|
61301
61439
|
}
|
|
61302
61440
|
function settleWithin(p3, ms) {
|
|
61303
61441
|
let timer;
|
|
61304
|
-
const timeout = new Promise((
|
|
61442
|
+
const timeout = new Promise((resolve17) => {
|
|
61305
61443
|
timer = setTimeout(() => {
|
|
61306
|
-
|
|
61444
|
+
resolve17("timeout");
|
|
61307
61445
|
}, ms);
|
|
61308
61446
|
timer.unref?.();
|
|
61309
61447
|
});
|
|
@@ -61358,9 +61496,9 @@ var SWITCH_OPEN_TIMEOUT_MS = 2e4;
|
|
|
61358
61496
|
async function openWithinTimeout(open, timeoutMs = SWITCH_OPEN_TIMEOUT_MS, dispose = disposeActive) {
|
|
61359
61497
|
const opening = open();
|
|
61360
61498
|
let timer;
|
|
61361
|
-
const timedOut = new Promise((
|
|
61499
|
+
const timedOut = new Promise((resolve17) => {
|
|
61362
61500
|
timer = setTimeout(() => {
|
|
61363
|
-
|
|
61501
|
+
resolve17("timeout");
|
|
61364
61502
|
}, timeoutMs);
|
|
61365
61503
|
timer.unref?.();
|
|
61366
61504
|
});
|
|
@@ -63031,6 +63169,65 @@ var chatCss = ` /* \u2500\u2500 Chat bubbles + tool pills \u2500\u2500\u2500\
|
|
|
63031
63169
|
}
|
|
63032
63170
|
`;
|
|
63033
63171
|
|
|
63172
|
+
// src/gui/app/styles/inline-import.ts
|
|
63173
|
+
var inlineImportCss = `
|
|
63174
|
+
/* \u2500\u2500 Inline import confirm card (assistant rail) \u2500\u2500 */
|
|
63175
|
+
.cd-sub { margin: 10px 0 6px; font-size: 12px; color: var(--text-muted, #9aa3ad); }
|
|
63176
|
+
.cd-row { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; margin-top: 8px; }
|
|
63177
|
+
.cd-path {
|
|
63178
|
+
flex: 1 1 220px; min-width: 0; box-sizing: border-box; height: 34px; padding: 0 10px;
|
|
63179
|
+
border-radius: 6px; border: 1px solid #2a2f36;
|
|
63180
|
+
background: var(--panel, #0e1116); color: var(--text, #e6e8eb); font-size: 13px;
|
|
63181
|
+
}
|
|
63182
|
+
.cd-status { margin-top: 12px; font-size: 13px; line-height: 1.5; }
|
|
63183
|
+
.cd-status.ok { color: #bef264; }
|
|
63184
|
+
.cd-status.err { color: #f87171; }
|
|
63185
|
+
.cd-status a { color: var(--accent, #bef264); }
|
|
63186
|
+
.cd-btn {
|
|
63187
|
+
height: 34px; padding: 0 14px; border-radius: 6px; border: 1px solid #2a2f36;
|
|
63188
|
+
background: transparent; color: var(--text, #e6e8eb); font-size: 13px;
|
|
63189
|
+
font-weight: 600; cursor: pointer;
|
|
63190
|
+
}
|
|
63191
|
+
.cd-btn:hover { background: rgba(255, 255, 255, 0.06); }
|
|
63192
|
+
.cd-btn.cd-primary { background: #bef264; color: #0b0d10; border-color: #bef264; }
|
|
63193
|
+
.cd-btn.cd-primary:hover { filter: brightness(1.06); }
|
|
63194
|
+
.cd-import-list { margin: 10px 0 0; padding-left: 18px; font-size: 13px; line-height: 1.6; }
|
|
63195
|
+
.cd-import-list li { margin: 2px 0; }
|
|
63196
|
+
.imp-sub { margin: 16px 0 6px; font-size: 13px; color: var(--text, #e6e8eb); }
|
|
63197
|
+
.imp-modes { display: flex; flex-direction: column; gap: 8px; margin: 0 0 6px; }
|
|
63198
|
+
.imp-modes label {
|
|
63199
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
63200
|
+
padding: 8px 10px; border: 1px solid #2a2f36; border-radius: 6px; cursor: pointer;
|
|
63201
|
+
}
|
|
63202
|
+
.imp-modes label:hover { background: rgba(255, 255, 255, 0.04); }
|
|
63203
|
+
.imp-modes input { margin-top: 2px; }
|
|
63204
|
+
.imp-modes b { color: var(--text, #e6e8eb); }
|
|
63205
|
+
.imp-percol {
|
|
63206
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
63207
|
+
margin: 8px 0 0; cursor: pointer; color: var(--text-dim, #aeb6c2);
|
|
63208
|
+
}
|
|
63209
|
+
.imp-percol input { margin-top: 2px; }
|
|
63210
|
+
.imp-match { border-left: 3px solid var(--accent, #7dd3fc); font-weight: 500; }
|
|
63211
|
+
.feed-item.import-confirm .imp-confirm-body { margin-top: 4px; }
|
|
63212
|
+
|
|
63213
|
+
/* \u2500\u2500 Live import progress in the card's log \u2500\u2500 */
|
|
63214
|
+
.feed-item.import-confirm .imp-card-log,
|
|
63215
|
+
.feed-item.import-live .imp-card-log {
|
|
63216
|
+
margin-top: 4px;
|
|
63217
|
+
font: 12px/1.6 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
|
63218
|
+
max-height: 200px; overflow-y: auto; color: var(--text-muted, #9aa3ad);
|
|
63219
|
+
}
|
|
63220
|
+
.imp-card-line { white-space: pre-wrap; word-break: break-word; }
|
|
63221
|
+
.imp-card-line.imp-done { color: var(--accent, #bef264); }
|
|
63222
|
+
.imp-card-line.imp-err { color: #f87171; }
|
|
63223
|
+
.imp-card-line.imp-spin::after {
|
|
63224
|
+
content: ''; display: inline-block; width: 10px; height: 10px; margin-left: 7px;
|
|
63225
|
+
border: 2px solid currentColor; border-right-color: transparent; border-radius: 50%;
|
|
63226
|
+
vertical-align: -1px; animation: imp-spin-kf 0.7s linear infinite;
|
|
63227
|
+
}
|
|
63228
|
+
@keyframes imp-spin-kf { to { transform: rotate(360deg); } }
|
|
63229
|
+
`;
|
|
63230
|
+
|
|
63034
63231
|
// src/gui/app/styles/index.ts
|
|
63035
63232
|
var css = [
|
|
63036
63233
|
tokensCss,
|
|
@@ -63052,7 +63249,8 @@ var css = [
|
|
|
63052
63249
|
fsWorkspaceCss,
|
|
63053
63250
|
settingsDrawerCss,
|
|
63054
63251
|
assistantRailCss,
|
|
63055
|
-
chatCss
|
|
63252
|
+
chatCss,
|
|
63253
|
+
inlineImportCss
|
|
63056
63254
|
].join("");
|
|
63057
63255
|
|
|
63058
63256
|
// src/gui/app/modules/display-config.ts
|
|
@@ -70623,6 +70821,11 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
70623
70821
|
// survivor if it was a duplicate). Multi-file drops do not navigate.
|
|
70624
70822
|
if (files.length === 1) {
|
|
70625
70823
|
uploadFile(files[0]).then(function (j) {
|
|
70824
|
+
// A structured source the server flagged as confirmable comes back with
|
|
70825
|
+
// an autoImport proposal \u2014 render the inline confirm card instead of
|
|
70826
|
+
// navigating to the file record. A silent import (autoImport.imported,
|
|
70827
|
+
// no reason) or a plain file keeps the open-the-record behavior.
|
|
70828
|
+
if (j && j.autoImport && j.autoImport.reason) { renderInlineImportCard(j.autoImport); return; }
|
|
70626
70829
|
if (j && (j.duplicateOf || j.id)) openSearchHit('files', j.duplicateOf || j.id);
|
|
70627
70830
|
});
|
|
70628
70831
|
return;
|
|
@@ -70632,7 +70835,15 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
70632
70835
|
var bar = ingestProgress(files.length);
|
|
70633
70836
|
var thunks = [];
|
|
70634
70837
|
for (var i = 0; i < files.length; i++) {
|
|
70635
|
-
(function (f) {
|
|
70838
|
+
(function (f) {
|
|
70839
|
+
thunks.push(function () {
|
|
70840
|
+
return uploadFile(f).then(function (j) {
|
|
70841
|
+
// A structured source within a batch still gets its own inline
|
|
70842
|
+
// confirm card (the batch as a whole does not navigate).
|
|
70843
|
+
if (j && j.autoImport && j.autoImport.reason) renderInlineImportCard(j.autoImport);
|
|
70844
|
+
});
|
|
70845
|
+
});
|
|
70846
|
+
})(files[i]);
|
|
70636
70847
|
}
|
|
70637
70848
|
runIngestBatch(thunks, INGEST_MAX_CONCURRENCY, bar.update).then(bar.done);
|
|
70638
70849
|
}
|
|
@@ -70788,6 +70999,237 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
70788
70999
|
})();
|
|
70789
71000
|
`;
|
|
70790
71001
|
|
|
71002
|
+
// src/gui/app/modules/inline-import.ts
|
|
71003
|
+
var inlineImportJs = `
|
|
71004
|
+
// \u2500\u2500 Inline structured-source import (confirm card in the assistant rail) \u2500\u2500
|
|
71005
|
+
function iiRailFeed() { return document.getElementById('rail-feed'); }
|
|
71006
|
+
function iiRailEmptyGone() {
|
|
71007
|
+
var e = document.getElementById('rail-empty');
|
|
71008
|
+
if (e) e.parentNode && e.parentNode.removeChild(e);
|
|
71009
|
+
}
|
|
71010
|
+
|
|
71011
|
+
// Read a newline-delimited-JSON response body, invoking onEvent(obj) per line.
|
|
71012
|
+
// Self-contained on purpose \u2014 this segment must not depend on any other.
|
|
71013
|
+
function iiStreamNdjson(url, payload, onEvent) {
|
|
71014
|
+
fetch(url, {
|
|
71015
|
+
method: 'POST',
|
|
71016
|
+
headers: { 'content-type': 'application/json' },
|
|
71017
|
+
body: JSON.stringify(payload),
|
|
71018
|
+
}).then(function (res) {
|
|
71019
|
+
if (!res.body || !res.body.getReader) {
|
|
71020
|
+
return res.text().then(function (t) {
|
|
71021
|
+
t.split('\\n').forEach(function (line) {
|
|
71022
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
71023
|
+
});
|
|
71024
|
+
});
|
|
71025
|
+
}
|
|
71026
|
+
var reader = res.body.getReader();
|
|
71027
|
+
var dec = new TextDecoder();
|
|
71028
|
+
var buf = '';
|
|
71029
|
+
function pump() {
|
|
71030
|
+
return reader.read().then(function (chunk) {
|
|
71031
|
+
if (chunk.done) {
|
|
71032
|
+
if (buf.trim()) { try { onEvent(JSON.parse(buf)); } catch (e) { /* skip */ } }
|
|
71033
|
+
return;
|
|
71034
|
+
}
|
|
71035
|
+
buf += dec.decode(chunk.value, { stream: true });
|
|
71036
|
+
var idx;
|
|
71037
|
+
while ((idx = buf.indexOf('\\n')) >= 0) {
|
|
71038
|
+
var line = buf.slice(0, idx);
|
|
71039
|
+
buf = buf.slice(idx + 1);
|
|
71040
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
71041
|
+
}
|
|
71042
|
+
return pump();
|
|
71043
|
+
});
|
|
71044
|
+
}
|
|
71045
|
+
return pump();
|
|
71046
|
+
}).catch(function (err) {
|
|
71047
|
+
onEvent({ phase: 'error', message: err && err.message ? err.message : 'Request failed' });
|
|
71048
|
+
});
|
|
71049
|
+
}
|
|
71050
|
+
|
|
71051
|
+
// Render the confirm card for a structured drop the server flagged as
|
|
71052
|
+
// needing confirmation. autoImport is the upload response's proposal:
|
|
71053
|
+
// { reason, fileId, plan:{entities,dimensions,linkages}, views, asOf,
|
|
71054
|
+
// asOfCandidates, asOfColumns, schemaMatch, matchedCount, totalEntities }.
|
|
71055
|
+
function renderInlineImportCard(autoImport) {
|
|
71056
|
+
if (!autoImport || !autoImport.fileId) return;
|
|
71057
|
+
var plan = autoImport.plan || {};
|
|
71058
|
+
var ents = plan.entities || [];
|
|
71059
|
+
var dims = plan.dimensions || [];
|
|
71060
|
+
var links = plan.linkages || [];
|
|
71061
|
+
var views = autoImport.views || [];
|
|
71062
|
+
var candidates = autoImport.asOfCandidates || [];
|
|
71063
|
+
var asOfColumns = autoImport.asOfColumns || [];
|
|
71064
|
+
var schemaMatch = autoImport.schemaMatch || {};
|
|
71065
|
+
var headerText = autoImport.reason === 'needs-confirm'
|
|
71066
|
+
? 'Add a dated snapshot'
|
|
71067
|
+
: 'Import as a new dataset';
|
|
71068
|
+
|
|
71069
|
+
iiRailEmptyGone();
|
|
71070
|
+
var feedEl = iiRailFeed();
|
|
71071
|
+
var card = document.createElement('div');
|
|
71072
|
+
card.className = 'feed-item import-confirm';
|
|
71073
|
+
var icon = document.createElement('div');
|
|
71074
|
+
icon.className = 'feed-icon';
|
|
71075
|
+
icon.textContent = '\u2913';
|
|
71076
|
+
var bodyEl = document.createElement('div');
|
|
71077
|
+
bodyEl.className = 'feed-body';
|
|
71078
|
+
var title = document.createElement('div');
|
|
71079
|
+
title.className = 'feed-summary';
|
|
71080
|
+
title.textContent = headerText;
|
|
71081
|
+
bodyEl.appendChild(title);
|
|
71082
|
+
|
|
71083
|
+
var parts = [];
|
|
71084
|
+
if (schemaMatch.isKnownDocument) {
|
|
71085
|
+
parts.push('<div class="cd-status ok imp-match">Recognized as a new period of an existing document — ' +
|
|
71086
|
+
schemaMatch.matchedCount + ' of ' + schemaMatch.totalEntities +
|
|
71087
|
+
' tables match what you already imported. It will be added as a dated snapshot.</div>');
|
|
71088
|
+
}
|
|
71089
|
+
parts.push('<div class="cd-status ok">Found ' + ents.length + ' entities, ' + dims.length +
|
|
71090
|
+
' dimensions, ' + links.length + ' links' +
|
|
71091
|
+
(views.length ? ', ' + views.length + ' reconstructed views (no duplicated rows)' : '') +
|
|
71092
|
+
'.</div><ul class="cd-import-list">');
|
|
71093
|
+
ents.forEach(function (e) {
|
|
71094
|
+
parts.push('<li><b>' + escapeHtml(e.name) + '</b> — ' + e.rowCount + ' rows, ' +
|
|
71095
|
+
(e.columns ? e.columns.length : 0) + ' cols · ' +
|
|
71096
|
+
(e.naturalKey ? 'key ' + escapeHtml(e.naturalKey) : 'keyless') + '</li>');
|
|
71097
|
+
});
|
|
71098
|
+
dims.forEach(function (d) {
|
|
71099
|
+
parts.push('<li><b>' + escapeHtml(d.name) + '</b> (dimension) — ' + d.distinctValues + ' values</li>');
|
|
71100
|
+
});
|
|
71101
|
+
views.forEach(function (v) {
|
|
71102
|
+
parts.push('<li><b>' + escapeHtml(v.name) + '</b> (view of ' + escapeHtml(v.master) + ' where ' +
|
|
71103
|
+
escapeHtml(v.filterColumn) + ' = ' + escapeHtml(String(v.filterValue)) + ') — ' +
|
|
71104
|
+
v.matchedRows + ' rows, not duplicated</li>');
|
|
71105
|
+
});
|
|
71106
|
+
parts.push('</ul>');
|
|
71107
|
+
|
|
71108
|
+
parts.push('<h4 class="imp-sub">As of date</h4>');
|
|
71109
|
+
var best = candidates[0];
|
|
71110
|
+
parts.push('<p class="cd-sub">' +
|
|
71111
|
+
(best ? 'Detected from ' + escapeHtml(best.evidence) + ' — edit if wrong.'
|
|
71112
|
+
: 'No date found in the file or its name — set the snapshot date, or leave blank to import undated.') +
|
|
71113
|
+
' A newer file is kept as a separate dated snapshot beside the prior one.</p>');
|
|
71114
|
+
parts.push('<div class="cd-row"><input class="cd-path" id="ii-asof" type="date" value="' + escapeHtml(autoImport.asOf || '') + '" aria-label="As of date" /></div>');
|
|
71115
|
+
if (candidates.length > 1) {
|
|
71116
|
+
parts.push('<div class="cd-sub">Other candidates: ' + candidates.slice(1, 5).map(function (c) {
|
|
71117
|
+
return '<a href="#" class="ii-asof-alt" data-date="' + escapeHtml(c.date) + '" title="' + escapeHtml(c.evidence) + '">' + escapeHtml(c.date) + '</a>';
|
|
71118
|
+
}).join(', ') + '</div>');
|
|
71119
|
+
}
|
|
71120
|
+
if (asOfColumns.length) {
|
|
71121
|
+
var colOpts = asOfColumns.slice(0, 6).map(function (c) {
|
|
71122
|
+
return '<option value="' + escapeHtml(c.column) + '" title="' + escapeHtml(c.evidence) + '">' +
|
|
71123
|
+
escapeHtml(c.column) + ' (' + escapeHtml(c.entity) + ', ' + c.distinctDates +
|
|
71124
|
+
' date' + (c.distinctDates === 1 ? '' : 's') + ')</option>';
|
|
71125
|
+
}).join('');
|
|
71126
|
+
parts.push('<label class="imp-percol"><input type="checkbox" id="ii-asof-percol"> ' +
|
|
71127
|
+
'<span>Date varies per row — use a date column instead (one file, many periods)</span></label>');
|
|
71128
|
+
parts.push('<div class="cd-row" id="ii-asof-col-row" style="display:none"><select class="cd-path" id="ii-asof-col">' + colOpts + '</select></div>');
|
|
71129
|
+
}
|
|
71130
|
+
|
|
71131
|
+
parts.push('<h4 class="imp-sub">What should Lattice bring in?</h4>');
|
|
71132
|
+
parts.push('<div class="imp-modes">' +
|
|
71133
|
+
'<label><input type="radio" name="ii-mode" value="both" checked> <span><b>Data model + contents</b> \u2014 the schema, the taxonomy, and all the rows.</span></label>' +
|
|
71134
|
+
'<label><input type="radio" name="ii-mode" value="schema"> <span><b>Data model / schema only</b> \u2014 tables, dimension values, and views. No rows.</span></label>' +
|
|
71135
|
+
'<label><input type="radio" name="ii-mode" value="contents"> <span><b>Contents only</b> \u2014 the rows and their links, into tables that already exist.</span></label>' +
|
|
71136
|
+
'</div>');
|
|
71137
|
+
parts.push('<div class="cd-row"><button class="cd-btn cd-primary" id="ii-apply" type="button">Import into Lattice</button></div>');
|
|
71138
|
+
parts.push('<div class="imp-card-log" id="ii-log"></div>');
|
|
71139
|
+
|
|
71140
|
+
var content = document.createElement('div');
|
|
71141
|
+
content.className = 'imp-confirm-body';
|
|
71142
|
+
content.innerHTML = parts.join('');
|
|
71143
|
+
bodyEl.appendChild(content);
|
|
71144
|
+
card.appendChild(icon);
|
|
71145
|
+
card.appendChild(bodyEl);
|
|
71146
|
+
if (feedEl) { feedEl.appendChild(card); feedEl.scrollTop = feedEl.scrollHeight; }
|
|
71147
|
+
|
|
71148
|
+
content.querySelectorAll('.ii-asof-alt').forEach(function (a) {
|
|
71149
|
+
a.addEventListener('click', function (e) {
|
|
71150
|
+
e.preventDefault();
|
|
71151
|
+
var input = document.getElementById('ii-asof');
|
|
71152
|
+
if (input) input.value = a.getAttribute('data-date') || '';
|
|
71153
|
+
});
|
|
71154
|
+
});
|
|
71155
|
+
var perCol = document.getElementById('ii-asof-percol');
|
|
71156
|
+
if (perCol) perCol.addEventListener('change', function () {
|
|
71157
|
+
var row = document.getElementById('ii-asof-col-row');
|
|
71158
|
+
var dateEl = document.getElementById('ii-asof');
|
|
71159
|
+
if (row) row.style.display = perCol.checked ? '' : 'none';
|
|
71160
|
+
if (dateEl) dateEl.disabled = perCol.checked;
|
|
71161
|
+
});
|
|
71162
|
+
|
|
71163
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
71164
|
+
if (applyBtn) applyBtn.addEventListener('click', function () {
|
|
71165
|
+
runInlineImport(autoImport.fileId, title, content);
|
|
71166
|
+
});
|
|
71167
|
+
}
|
|
71168
|
+
|
|
71169
|
+
// POST the confirmed proposal to /api/import/apply and stream the pipeline
|
|
71170
|
+
// live into the card's log. On 'done' show a success summary + refresh the
|
|
71171
|
+
// Objects nav in place; on 'error' show the message.
|
|
71172
|
+
function runInlineImport(fileId, title, content) {
|
|
71173
|
+
var sel = content.querySelector('input[name="ii-mode"]:checked');
|
|
71174
|
+
var mode = sel ? sel.value : 'both';
|
|
71175
|
+
var asofEl = document.getElementById('ii-asof');
|
|
71176
|
+
var asOf = asofEl ? asofEl.value : '';
|
|
71177
|
+
var perColEl = document.getElementById('ii-asof-percol');
|
|
71178
|
+
var colSel = document.getElementById('ii-asof-col');
|
|
71179
|
+
var asOfColumn = (perColEl && perColEl.checked && colSel) ? colSel.value : '';
|
|
71180
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
71181
|
+
if (applyBtn) applyBtn.disabled = true;
|
|
71182
|
+
|
|
71183
|
+
var feedEl = iiRailFeed();
|
|
71184
|
+
var log = document.getElementById('ii-log');
|
|
71185
|
+
function addLine(text, cls) {
|
|
71186
|
+
if (!log) return null;
|
|
71187
|
+
var d = document.createElement('div');
|
|
71188
|
+
d.className = 'imp-card-line' + (cls ? ' ' + cls : '');
|
|
71189
|
+
d.textContent = text;
|
|
71190
|
+
log.appendChild(d);
|
|
71191
|
+
while (log.childNodes.length > 60) log.removeChild(log.firstChild);
|
|
71192
|
+
log.scrollTop = log.scrollHeight;
|
|
71193
|
+
if (feedEl) feedEl.scrollTop = feedEl.scrollHeight;
|
|
71194
|
+
return d;
|
|
71195
|
+
}
|
|
71196
|
+
title.textContent = 'Importing your data\u2026';
|
|
71197
|
+
addLine('Starting\u2026');
|
|
71198
|
+
|
|
71199
|
+
iiStreamNdjson('/api/import/apply', { fileId: fileId, mode: mode, asOf: asOf, asOfColumn: asOfColumn }, function (evt) {
|
|
71200
|
+
if (!evt) return;
|
|
71201
|
+
if (evt.phase === 'done') {
|
|
71202
|
+
var r = evt.result || {};
|
|
71203
|
+
var rbt = r.rowsByTable || {};
|
|
71204
|
+
var names = Object.keys(rbt);
|
|
71205
|
+
var total = 0;
|
|
71206
|
+
names.forEach(function (n) { total += (rbt[n] || 0); });
|
|
71207
|
+
title.textContent = 'Imported ' + names.length + ' tables' + (mode === 'schema' ? '' : ', ' + total + ' rows');
|
|
71208
|
+
var upd = addLine('Updating your objects\u2026', 'imp-spin');
|
|
71209
|
+
refreshEntities().then(function () {
|
|
71210
|
+
renderSidebar();
|
|
71211
|
+
renderRoute();
|
|
71212
|
+
var count = (state.entities && state.entities.tables) ? state.entities.tables.length : names.length;
|
|
71213
|
+
if (upd) {
|
|
71214
|
+
upd.className = 'imp-card-line imp-done';
|
|
71215
|
+
upd.textContent = '\u2713 Done \u2014 ' + count + ' objects in your workspace';
|
|
71216
|
+
}
|
|
71217
|
+
}).catch(function () {
|
|
71218
|
+
if (upd) {
|
|
71219
|
+
upd.className = 'imp-card-line imp-err';
|
|
71220
|
+
upd.textContent = 'Imported, but refreshing the view failed \u2014 reload to see your objects.';
|
|
71221
|
+
}
|
|
71222
|
+
});
|
|
71223
|
+
} else if (evt.phase === 'error') {
|
|
71224
|
+
title.textContent = 'Import failed';
|
|
71225
|
+
addLine('Error: ' + (evt.message || 'import failed'), 'imp-err');
|
|
71226
|
+
} else if (evt.message) {
|
|
71227
|
+
addLine(evt.message);
|
|
71228
|
+
}
|
|
71229
|
+
});
|
|
71230
|
+
}
|
|
71231
|
+
`;
|
|
71232
|
+
|
|
70791
71233
|
// src/gui/app/modules/index.ts
|
|
70792
71234
|
var appJs = [
|
|
70793
71235
|
displayConfigJs,
|
|
@@ -70816,7 +71258,8 @@ var appJs = [
|
|
|
70816
71258
|
dataModelJs,
|
|
70817
71259
|
latticeTeamsJs,
|
|
70818
71260
|
onboardingJs,
|
|
70819
|
-
createDatabaseWizardJs
|
|
71261
|
+
createDatabaseWizardJs,
|
|
71262
|
+
inlineImportJs
|
|
70820
71263
|
].join("");
|
|
70821
71264
|
|
|
70822
71265
|
// src/gui/app/analytics.ts
|
|
@@ -73191,19 +73634,1172 @@ async function dispatchChatRoute(req, res, ctx) {
|
|
|
73191
73634
|
}
|
|
73192
73635
|
|
|
73193
73636
|
// src/gui/ingest-routes.ts
|
|
73194
|
-
var
|
|
73637
|
+
var import_node_fs40 = require("fs");
|
|
73195
73638
|
var import_promises12 = require("fs/promises");
|
|
73196
73639
|
var import_node_os9 = require("os");
|
|
73197
|
-
var
|
|
73640
|
+
var import_node_path42 = require("path");
|
|
73198
73641
|
init_mutations();
|
|
73199
73642
|
init_extract();
|
|
73200
|
-
var
|
|
73643
|
+
var import_node_crypto23 = require("crypto");
|
|
73201
73644
|
init_assistant_routes();
|
|
73202
73645
|
init_http2();
|
|
73203
73646
|
init_enrich();
|
|
73204
73647
|
init_ingest_url();
|
|
73205
73648
|
init_file_row();
|
|
73206
73649
|
init_dedup_service();
|
|
73650
|
+
|
|
73651
|
+
// src/gui/import-auto.ts
|
|
73652
|
+
var import_node_fs39 = require("fs");
|
|
73653
|
+
|
|
73654
|
+
// src/import/infer.ts
|
|
73655
|
+
var SAMPLE = 300;
|
|
73656
|
+
var PREFERRED_KEYS = ["code", "id", "slug", "key", "ticker", "symbol"];
|
|
73657
|
+
var NEVER_KEY = /* @__PURE__ */ new Set([
|
|
73658
|
+
"description",
|
|
73659
|
+
"notes",
|
|
73660
|
+
"summary",
|
|
73661
|
+
"desc",
|
|
73662
|
+
"comment",
|
|
73663
|
+
"comments",
|
|
73664
|
+
"bio",
|
|
73665
|
+
"text",
|
|
73666
|
+
"body"
|
|
73667
|
+
]);
|
|
73668
|
+
var FREETEXT = /* @__PURE__ */ new Set([...NEVER_KEY, "name", "title", "company", "label"]);
|
|
73669
|
+
var DIM_MAX_DISTINCT = 64;
|
|
73670
|
+
var DIM_MAX_RATIO = 0.5;
|
|
73671
|
+
var LINK_MIN_CONFIDENCE = 0.3;
|
|
73672
|
+
function isPlainObject(v2) {
|
|
73673
|
+
return typeof v2 === "object" && v2 !== null && !Array.isArray(v2);
|
|
73674
|
+
}
|
|
73675
|
+
function sourceRecords(data, entity) {
|
|
73676
|
+
const v2 = data[entity.sourceKey];
|
|
73677
|
+
if (!Array.isArray(v2)) return [];
|
|
73678
|
+
if (entity.columnar) {
|
|
73679
|
+
const cols = data[entity.sourceKey + "Cols"];
|
|
73680
|
+
if (!Array.isArray(cols)) return [];
|
|
73681
|
+
return v2.map((row) => {
|
|
73682
|
+
const o3 = {};
|
|
73683
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
73684
|
+
return o3;
|
|
73685
|
+
});
|
|
73686
|
+
}
|
|
73687
|
+
return v2.filter(isPlainObject);
|
|
73688
|
+
}
|
|
73689
|
+
function normalizeName(key) {
|
|
73690
|
+
const s2 = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
73691
|
+
if (!s2) return "field";
|
|
73692
|
+
return /^[a-z]/.test(s2) ? s2 : "f_" + s2;
|
|
73693
|
+
}
|
|
73694
|
+
var ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
73695
|
+
var ISO_DATETIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/;
|
|
73696
|
+
function inferFieldType(values) {
|
|
73697
|
+
const present = values.filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
73698
|
+
if (present.length === 0) return "text";
|
|
73699
|
+
if (present.every((v2) => typeof v2 === "number")) {
|
|
73700
|
+
return present.every((v2) => Number.isInteger(v2)) ? "integer" : "real";
|
|
73701
|
+
}
|
|
73702
|
+
if (present.every((v2) => typeof v2 === "boolean")) return "boolean";
|
|
73703
|
+
if (present.every((v2) => typeof v2 === "string")) {
|
|
73704
|
+
if (present.every((v2) => ISO_DATE.test(v2))) return "date";
|
|
73705
|
+
if (present.every((v2) => ISO_DATETIME.test(v2))) return "datetime";
|
|
73706
|
+
}
|
|
73707
|
+
return "text";
|
|
73708
|
+
}
|
|
73709
|
+
function norm2(v2) {
|
|
73710
|
+
return String(v2).trim().toLowerCase();
|
|
73711
|
+
}
|
|
73712
|
+
function isNumericValue(v2) {
|
|
73713
|
+
if (typeof v2 === "number") return Number.isFinite(v2);
|
|
73714
|
+
if (typeof v2 !== "string") return false;
|
|
73715
|
+
const s2 = v2.replace(/[\s,$%()]/g, "");
|
|
73716
|
+
return s2 !== "" && Number.isFinite(Number(s2));
|
|
73717
|
+
}
|
|
73718
|
+
function profileColumns(records) {
|
|
73719
|
+
const keys = /* @__PURE__ */ new Set();
|
|
73720
|
+
for (const r6 of records.slice(0, SAMPLE)) for (const k6 of Object.keys(r6)) keys.add(k6);
|
|
73721
|
+
const out = /* @__PURE__ */ new Map();
|
|
73722
|
+
for (const key of keys) {
|
|
73723
|
+
let isArray = false;
|
|
73724
|
+
const sample = [];
|
|
73725
|
+
const valueSet = /* @__PURE__ */ new Set();
|
|
73726
|
+
const distinctSet = /* @__PURE__ */ new Set();
|
|
73727
|
+
let nonNull = 0;
|
|
73728
|
+
let numeric = 0;
|
|
73729
|
+
for (const r6 of records) {
|
|
73730
|
+
const v2 = r6[key];
|
|
73731
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
73732
|
+
nonNull++;
|
|
73733
|
+
if (Array.isArray(v2)) {
|
|
73734
|
+
isArray = true;
|
|
73735
|
+
for (const e6 of v2) {
|
|
73736
|
+
if (e6 !== null && e6 !== void 0 && e6 !== "") {
|
|
73737
|
+
valueSet.add(norm2(e6));
|
|
73738
|
+
distinctSet.add(norm2(e6));
|
|
73739
|
+
}
|
|
73740
|
+
}
|
|
73741
|
+
} else {
|
|
73742
|
+
if (sample.length < SAMPLE) sample.push(v2);
|
|
73743
|
+
if (typeof v2 === "string") valueSet.add(norm2(v2));
|
|
73744
|
+
distinctSet.add(norm2(v2));
|
|
73745
|
+
if (isNumericValue(v2)) numeric++;
|
|
73746
|
+
}
|
|
73747
|
+
}
|
|
73748
|
+
out.set(key, {
|
|
73749
|
+
sourceKey: key,
|
|
73750
|
+
isArray,
|
|
73751
|
+
type: isArray ? "text" : inferFieldType(sample),
|
|
73752
|
+
// Cardinality counts ALL distinct values (numbers + strings). Counting only
|
|
73753
|
+
// string values let a mostly-numeric column with a few text sentinels (e.g.
|
|
73754
|
+
// a "TEV/EBITDA" of numbers + "NM") look low-cardinality and slip in as a
|
|
73755
|
+
// junk dimension.
|
|
73756
|
+
distinct: distinctSet.size,
|
|
73757
|
+
valueSet,
|
|
73758
|
+
numericFraction: nonNull > 0 ? numeric / nonNull : 0
|
|
73759
|
+
});
|
|
73760
|
+
}
|
|
73761
|
+
return out;
|
|
73762
|
+
}
|
|
73763
|
+
function pickNaturalKey(records, profiles) {
|
|
73764
|
+
const n3 = records.length;
|
|
73765
|
+
const isUnique = (key) => {
|
|
73766
|
+
const seen = /* @__PURE__ */ new Set();
|
|
73767
|
+
for (const r6 of records) {
|
|
73768
|
+
const v2 = r6[key];
|
|
73769
|
+
if (v2 === null || v2 === void 0 || v2 === "") return false;
|
|
73770
|
+
const k6 = norm2(v2);
|
|
73771
|
+
if (seen.has(k6)) return false;
|
|
73772
|
+
seen.add(k6);
|
|
73773
|
+
}
|
|
73774
|
+
return seen.size === n3;
|
|
73775
|
+
};
|
|
73776
|
+
for (const pref of PREFERRED_KEYS) {
|
|
73777
|
+
for (const [key, p3] of profiles) {
|
|
73778
|
+
if (p3.isArray) continue;
|
|
73779
|
+
if (normalizeName(key) === pref && isUnique(key)) return key;
|
|
73780
|
+
}
|
|
73781
|
+
}
|
|
73782
|
+
for (const [key, p3] of profiles) {
|
|
73783
|
+
if (p3.isArray) continue;
|
|
73784
|
+
if (NEVER_KEY.has(normalizeName(key))) continue;
|
|
73785
|
+
if ((p3.type === "text" || p3.type === "integer") && isUnique(key)) return key;
|
|
73786
|
+
}
|
|
73787
|
+
return null;
|
|
73788
|
+
}
|
|
73789
|
+
function inferSchema(data, opts = {}) {
|
|
73790
|
+
const skipped = [];
|
|
73791
|
+
const consumedColsKeys = /* @__PURE__ */ new Set();
|
|
73792
|
+
for (const key of Object.keys(data)) {
|
|
73793
|
+
const v2 = data[key];
|
|
73794
|
+
const cols = data[key + "Cols"];
|
|
73795
|
+
if (Array.isArray(v2) && v2.length > 0 && Array.isArray(v2[0]) && Array.isArray(cols) && cols.every((c6) => typeof c6 === "string")) {
|
|
73796
|
+
consumedColsKeys.add(key + "Cols");
|
|
73797
|
+
}
|
|
73798
|
+
}
|
|
73799
|
+
const sources = [];
|
|
73800
|
+
for (const key of Object.keys(data)) {
|
|
73801
|
+
if (consumedColsKeys.has(key)) continue;
|
|
73802
|
+
const v2 = data[key];
|
|
73803
|
+
if (!Array.isArray(v2) || v2.length === 0) {
|
|
73804
|
+
skipped.push({
|
|
73805
|
+
key,
|
|
73806
|
+
reason: isPlainObject(v2) ? "object (derived/rollup)" : "scalar/empty (meta or derived)"
|
|
73807
|
+
});
|
|
73808
|
+
continue;
|
|
73809
|
+
}
|
|
73810
|
+
let records;
|
|
73811
|
+
let columnar = false;
|
|
73812
|
+
if (isPlainObject(v2[0])) {
|
|
73813
|
+
records = v2.filter(isPlainObject);
|
|
73814
|
+
} else if (Array.isArray(v2[0]) && Array.isArray(data[key + "Cols"])) {
|
|
73815
|
+
const cols = data[key + "Cols"];
|
|
73816
|
+
records = v2.map((row) => {
|
|
73817
|
+
const o3 = {};
|
|
73818
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
73819
|
+
return o3;
|
|
73820
|
+
});
|
|
73821
|
+
columnar = true;
|
|
73822
|
+
} else {
|
|
73823
|
+
skipped.push({ key, reason: "array of scalars (not a record set)" });
|
|
73824
|
+
continue;
|
|
73825
|
+
}
|
|
73826
|
+
const name = opts.rename?.[key] ?? normalizeName(key);
|
|
73827
|
+
const profiles = profileColumns(records);
|
|
73828
|
+
sources.push({
|
|
73829
|
+
name,
|
|
73830
|
+
sourceKey: key,
|
|
73831
|
+
records,
|
|
73832
|
+
columnar,
|
|
73833
|
+
profiles,
|
|
73834
|
+
naturalKey: pickNaturalKey(records, profiles)
|
|
73835
|
+
});
|
|
73836
|
+
}
|
|
73837
|
+
const linkages = [];
|
|
73838
|
+
const consumedFields = /* @__PURE__ */ new Map();
|
|
73839
|
+
const linkedTargets = /* @__PURE__ */ new Map();
|
|
73840
|
+
const consume = (e6, f6) => {
|
|
73841
|
+
let set = consumedFields.get(e6);
|
|
73842
|
+
if (!set) {
|
|
73843
|
+
set = /* @__PURE__ */ new Set();
|
|
73844
|
+
consumedFields.set(e6, set);
|
|
73845
|
+
}
|
|
73846
|
+
set.add(f6);
|
|
73847
|
+
};
|
|
73848
|
+
const markTarget = (e6, t8) => {
|
|
73849
|
+
let set = linkedTargets.get(e6);
|
|
73850
|
+
if (!set) {
|
|
73851
|
+
set = /* @__PURE__ */ new Set();
|
|
73852
|
+
linkedTargets.set(e6, set);
|
|
73853
|
+
}
|
|
73854
|
+
set.add(t8);
|
|
73855
|
+
};
|
|
73856
|
+
function bestTarget(self, values) {
|
|
73857
|
+
if (values.size === 0) return null;
|
|
73858
|
+
let best = null;
|
|
73859
|
+
for (const t8 of sources) {
|
|
73860
|
+
if (t8.name === self.name || !t8.naturalKey) continue;
|
|
73861
|
+
const p3 = t8.profiles.get(t8.naturalKey);
|
|
73862
|
+
if (!p3 || p3.valueSet.size === 0) continue;
|
|
73863
|
+
let matched = 0;
|
|
73864
|
+
for (const v2 of values) if (p3.valueSet.has(v2)) matched++;
|
|
73865
|
+
if (matched > 0 && (best === null || matched > best.matched)) {
|
|
73866
|
+
best = { target: t8, column: t8.naturalKey, matched };
|
|
73867
|
+
}
|
|
73868
|
+
}
|
|
73869
|
+
return best;
|
|
73870
|
+
}
|
|
73871
|
+
for (const pass of ["array", "scalar"]) {
|
|
73872
|
+
for (const e6 of sources) {
|
|
73873
|
+
for (const [field, p3] of e6.profiles) {
|
|
73874
|
+
if (pass === "array" ? !p3.isArray : p3.isArray) continue;
|
|
73875
|
+
if (pass === "scalar") {
|
|
73876
|
+
if (field === e6.naturalKey) continue;
|
|
73877
|
+
if (FREETEXT.has(normalizeName(field)) || NEVER_KEY.has(normalizeName(field))) continue;
|
|
73878
|
+
if (p3.type !== "text") continue;
|
|
73879
|
+
}
|
|
73880
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
73881
|
+
const best = bestTarget(e6, p3.valueSet);
|
|
73882
|
+
if (!best) continue;
|
|
73883
|
+
const confidence = best.matched / p3.valueSet.size;
|
|
73884
|
+
if (confidence < LINK_MIN_CONFIDENCE) continue;
|
|
73885
|
+
if (linkedTargets.get(e6.name)?.has(best.target.name)) {
|
|
73886
|
+
consume(e6.name, field);
|
|
73887
|
+
continue;
|
|
73888
|
+
}
|
|
73889
|
+
const link = {
|
|
73890
|
+
kind: pass === "array" ? "many-to-many" : "many-to-one",
|
|
73891
|
+
fromEntity: e6.name,
|
|
73892
|
+
fromField: field,
|
|
73893
|
+
toEntity: best.target.name,
|
|
73894
|
+
toKey: normalizeName(best.column),
|
|
73895
|
+
matched: best.matched,
|
|
73896
|
+
unresolved: p3.valueSet.size - best.matched,
|
|
73897
|
+
confidence
|
|
73898
|
+
};
|
|
73899
|
+
if (pass === "array") link.junction = `${e6.name}_${best.target.name}`;
|
|
73900
|
+
linkages.push(link);
|
|
73901
|
+
consume(e6.name, field);
|
|
73902
|
+
markTarget(e6.name, best.target.name);
|
|
73903
|
+
}
|
|
73904
|
+
}
|
|
73905
|
+
}
|
|
73906
|
+
const dimColumnNames = /* @__PURE__ */ new Map();
|
|
73907
|
+
for (const e6 of sources) {
|
|
73908
|
+
for (const [field, p3] of e6.profiles) {
|
|
73909
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
73910
|
+
const nn = normalizeName(field);
|
|
73911
|
+
let arr = dimColumnNames.get(nn);
|
|
73912
|
+
if (!arr) {
|
|
73913
|
+
arr = [];
|
|
73914
|
+
dimColumnNames.set(nn, arr);
|
|
73915
|
+
}
|
|
73916
|
+
arr.push(e6);
|
|
73917
|
+
}
|
|
73918
|
+
}
|
|
73919
|
+
const dimensions = [];
|
|
73920
|
+
const dimByName = /* @__PURE__ */ new Map();
|
|
73921
|
+
for (const e6 of sources) {
|
|
73922
|
+
for (const [field, p3] of e6.profiles) {
|
|
73923
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
73924
|
+
if (field === e6.naturalKey) continue;
|
|
73925
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
73926
|
+
const nn = normalizeName(field);
|
|
73927
|
+
if (FREETEXT.has(nn)) continue;
|
|
73928
|
+
const ratio = p3.distinct / Math.max(1, e6.records.length);
|
|
73929
|
+
const sharedAcross = dimColumnNames.get(nn)?.length ?? 1;
|
|
73930
|
+
const isDim = p3.distinct >= 1 && p3.distinct <= DIM_MAX_DISTINCT && (ratio <= DIM_MAX_RATIO || sharedAcross >= 2);
|
|
73931
|
+
if (!isDim) continue;
|
|
73932
|
+
let dim = dimByName.get(nn);
|
|
73933
|
+
if (!dim) {
|
|
73934
|
+
dim = { name: nn, sourceField: field, fromEntities: [], distinctValues: 0 };
|
|
73935
|
+
dimByName.set(nn, dim);
|
|
73936
|
+
dimensions.push(dim);
|
|
73937
|
+
}
|
|
73938
|
+
if (!dim.fromEntities.includes(e6.name)) dim.fromEntities.push(e6.name);
|
|
73939
|
+
linkages.push({
|
|
73940
|
+
kind: "dimension",
|
|
73941
|
+
fromEntity: e6.name,
|
|
73942
|
+
fromField: field,
|
|
73943
|
+
toEntity: nn,
|
|
73944
|
+
toKey: "value",
|
|
73945
|
+
junction: `${e6.name}_${nn}`,
|
|
73946
|
+
matched: p3.distinct,
|
|
73947
|
+
unresolved: 0,
|
|
73948
|
+
confidence: 1
|
|
73949
|
+
});
|
|
73950
|
+
consume(e6.name, field);
|
|
73951
|
+
}
|
|
73952
|
+
}
|
|
73953
|
+
for (const dim of dimensions) {
|
|
73954
|
+
const all = /* @__PURE__ */ new Set();
|
|
73955
|
+
for (const name of dim.fromEntities) {
|
|
73956
|
+
const e6 = sources.find((s2) => s2.name === name);
|
|
73957
|
+
if (!e6) continue;
|
|
73958
|
+
for (const [f6, p3] of e6.profiles) {
|
|
73959
|
+
if (normalizeName(f6) === dim.name) for (const v2 of p3.valueSet) all.add(v2);
|
|
73960
|
+
}
|
|
73961
|
+
}
|
|
73962
|
+
dim.distinctValues = all.size;
|
|
73963
|
+
}
|
|
73964
|
+
const entities = sources.map((e6) => {
|
|
73965
|
+
const columns = [];
|
|
73966
|
+
for (const [field, p3] of e6.profiles) {
|
|
73967
|
+
if (p3.isArray) continue;
|
|
73968
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
73969
|
+
columns.push({ name: normalizeName(field), sourceKey: field, type: p3.type });
|
|
73970
|
+
}
|
|
73971
|
+
return {
|
|
73972
|
+
name: e6.name,
|
|
73973
|
+
sourceKey: e6.sourceKey,
|
|
73974
|
+
columns,
|
|
73975
|
+
naturalKey: e6.naturalKey ? normalizeName(e6.naturalKey) : null,
|
|
73976
|
+
naturalKeySource: e6.naturalKey,
|
|
73977
|
+
rowCount: e6.records.length,
|
|
73978
|
+
columnar: e6.columnar
|
|
73979
|
+
};
|
|
73980
|
+
});
|
|
73981
|
+
return { entities, dimensions, linkages, skipped };
|
|
73982
|
+
}
|
|
73983
|
+
|
|
73984
|
+
// src/import/dedupe-views.ts
|
|
73985
|
+
init_normalize();
|
|
73986
|
+
var SAMPLE2 = 300;
|
|
73987
|
+
var VIEW_MIN_OVERLAP = 0.8;
|
|
73988
|
+
function buildEntityData(plan, data) {
|
|
73989
|
+
return plan.entities.map((e6) => {
|
|
73990
|
+
const records = sourceRecords(data, e6);
|
|
73991
|
+
const colSet = /* @__PURE__ */ new Set();
|
|
73992
|
+
const colSource = /* @__PURE__ */ new Map();
|
|
73993
|
+
for (const r6 of records.slice(0, SAMPLE2)) {
|
|
73994
|
+
for (const k6 of Object.keys(r6)) {
|
|
73995
|
+
const n3 = normalizeName(k6);
|
|
73996
|
+
colSet.add(n3);
|
|
73997
|
+
if (!colSource.has(n3)) colSource.set(n3, k6);
|
|
73998
|
+
}
|
|
73999
|
+
}
|
|
74000
|
+
const normRows = records.map((r6) => {
|
|
74001
|
+
const o3 = {};
|
|
74002
|
+
for (const k6 of Object.keys(r6)) o3[normalizeName(k6)] = r6[k6];
|
|
74003
|
+
return o3;
|
|
74004
|
+
});
|
|
74005
|
+
return { name: e6.name, sourceKey: e6.sourceKey, cols: [...colSet], colSource, normRows };
|
|
74006
|
+
});
|
|
74007
|
+
}
|
|
74008
|
+
function pickIdentity(a6, shared) {
|
|
74009
|
+
let bestCol = null;
|
|
74010
|
+
let bestDistinct = -1;
|
|
74011
|
+
for (const c6 of shared) {
|
|
74012
|
+
const vals = /* @__PURE__ */ new Set();
|
|
74013
|
+
let textish = 0;
|
|
74014
|
+
let total = 0;
|
|
74015
|
+
for (const r6 of a6.normRows) {
|
|
74016
|
+
const v2 = r6[c6];
|
|
74017
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
74018
|
+
total++;
|
|
74019
|
+
if (typeof v2 === "string") textish++;
|
|
74020
|
+
vals.add(normalizeText(v2));
|
|
74021
|
+
}
|
|
74022
|
+
if (total === 0 || textish / total < 0.7) continue;
|
|
74023
|
+
if (vals.size > bestDistinct) {
|
|
74024
|
+
bestDistinct = vals.size;
|
|
74025
|
+
bestCol = c6;
|
|
74026
|
+
}
|
|
74027
|
+
}
|
|
74028
|
+
return bestCol;
|
|
74029
|
+
}
|
|
74030
|
+
function dedupeAndDetectViews(plan, data) {
|
|
74031
|
+
const entities = buildEntityData(plan, data);
|
|
74032
|
+
const views = [];
|
|
74033
|
+
const asView = /* @__PURE__ */ new Set();
|
|
74034
|
+
const colKeeps = [];
|
|
74035
|
+
for (const a6 of entities) {
|
|
74036
|
+
if (a6.cols.length < 2 || a6.normRows.length === 0) continue;
|
|
74037
|
+
const tabName = normalizeText(a6.sourceKey);
|
|
74038
|
+
if (!tabName) continue;
|
|
74039
|
+
const aColSet = new Set(a6.cols);
|
|
74040
|
+
let best = null;
|
|
74041
|
+
for (const b6 of entities) {
|
|
74042
|
+
if (b6.name === a6.name || asView.has(b6.name)) continue;
|
|
74043
|
+
if (b6.normRows.length < a6.normRows.length) continue;
|
|
74044
|
+
const bColSet = new Set(b6.cols);
|
|
74045
|
+
const shared = a6.cols.filter((c6) => bColSet.has(c6));
|
|
74046
|
+
if (shared.length < Math.max(2, Math.ceil(a6.cols.length * 0.5))) continue;
|
|
74047
|
+
const identity = pickIdentity(a6, shared);
|
|
74048
|
+
if (!identity) continue;
|
|
74049
|
+
const aIds = new Set(
|
|
74050
|
+
a6.normRows.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== "")
|
|
74051
|
+
);
|
|
74052
|
+
if (aIds.size === 0) continue;
|
|
74053
|
+
for (const disc of b6.cols) {
|
|
74054
|
+
if (aColSet.has(disc)) continue;
|
|
74055
|
+
const sub = b6.normRows.filter((r6) => normalizeText(r6[disc]) === tabName);
|
|
74056
|
+
if (sub.length === 0) continue;
|
|
74057
|
+
const bIds = new Set(sub.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== ""));
|
|
74058
|
+
let inter = 0;
|
|
74059
|
+
for (const id of aIds) if (bIds.has(id)) inter++;
|
|
74060
|
+
const overlap = inter / aIds.size;
|
|
74061
|
+
if (overlap < VIEW_MIN_OVERLAP) continue;
|
|
74062
|
+
const rawRow = sub.find((r6) => typeof r6[disc] === "string" || typeof r6[disc] === "number");
|
|
74063
|
+
const raw = rawRow ? rawRow[disc] : void 0;
|
|
74064
|
+
if (typeof raw !== "string" && typeof raw !== "number") continue;
|
|
74065
|
+
if (best === null || overlap > best.overlap || overlap === best.overlap && b6.cols.length > best.master.cols.length) {
|
|
74066
|
+
best = { master: b6, disc, value: String(raw), matched: sub.length, overlap };
|
|
74067
|
+
}
|
|
74068
|
+
}
|
|
74069
|
+
}
|
|
74070
|
+
if (!best) continue;
|
|
74071
|
+
views.push({
|
|
74072
|
+
name: a6.name,
|
|
74073
|
+
master: best.master.name,
|
|
74074
|
+
filterColumn: best.disc,
|
|
74075
|
+
filterValue: best.value,
|
|
74076
|
+
matchedRows: best.matched
|
|
74077
|
+
});
|
|
74078
|
+
asView.add(a6.name);
|
|
74079
|
+
colKeeps.push({ master: best.master, col: best.disc });
|
|
74080
|
+
}
|
|
74081
|
+
for (const { master, col } of colKeeps) {
|
|
74082
|
+
const masterEntity = plan.entities.find((e6) => e6.name === master.name);
|
|
74083
|
+
if (!masterEntity || masterEntity.columns.some((c6) => c6.name === col)) continue;
|
|
74084
|
+
masterEntity.columns.push({
|
|
74085
|
+
name: col,
|
|
74086
|
+
sourceKey: master.colSource.get(col) ?? col,
|
|
74087
|
+
type: inferFieldType(master.normRows.map((r6) => r6[col]))
|
|
74088
|
+
});
|
|
74089
|
+
}
|
|
74090
|
+
if (views.length === 0) return { plan, views };
|
|
74091
|
+
const nextPlan = {
|
|
74092
|
+
entities: plan.entities.filter((e6) => !asView.has(e6.name)),
|
|
74093
|
+
linkages: plan.linkages.filter((l4) => !asView.has(l4.fromEntity)),
|
|
74094
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.filter((n3) => !asView.has(n3)) })).filter((d6) => d6.fromEntities.length > 0),
|
|
74095
|
+
skipped: plan.skipped
|
|
74096
|
+
};
|
|
74097
|
+
return { plan: nextPlan, views };
|
|
74098
|
+
}
|
|
74099
|
+
|
|
74100
|
+
// src/import/excel.ts
|
|
74101
|
+
var import_node_path40 = require("path");
|
|
74102
|
+
var HEADER_SCAN_ROWS = 25;
|
|
74103
|
+
function cellValue(v2) {
|
|
74104
|
+
if (v2 === null || v2 === void 0) return null;
|
|
74105
|
+
if (v2 instanceof Date) return v2.toISOString().slice(0, 10);
|
|
74106
|
+
if (typeof v2 === "object") {
|
|
74107
|
+
const o3 = v2;
|
|
74108
|
+
if ("result" in o3) return cellValue(o3.result);
|
|
74109
|
+
if ("text" in o3) return o3.text;
|
|
74110
|
+
if ("richText" in o3 && Array.isArray(o3.richText)) {
|
|
74111
|
+
return o3.richText.map((t8) => t8.text ?? "").join("");
|
|
74112
|
+
}
|
|
74113
|
+
return null;
|
|
74114
|
+
}
|
|
74115
|
+
return v2;
|
|
74116
|
+
}
|
|
74117
|
+
function isFilled(v2) {
|
|
74118
|
+
return v2 !== null && v2 !== void 0 && v2 !== "";
|
|
74119
|
+
}
|
|
74120
|
+
function sheetToRecords(ws) {
|
|
74121
|
+
const rowCount = ws.rowCount;
|
|
74122
|
+
const colCount = ws.columnCount;
|
|
74123
|
+
if (rowCount < 2 || colCount < 2) return [];
|
|
74124
|
+
const nonEmpty = (r6) => {
|
|
74125
|
+
let n3 = 0;
|
|
74126
|
+
for (let c6 = 1; c6 <= colCount; c6++) if (isFilled(cellValue(ws.getCell(r6, c6).value))) n3++;
|
|
74127
|
+
return n3;
|
|
74128
|
+
};
|
|
74129
|
+
const threshold = Math.max(3, Math.floor(colCount * 0.4));
|
|
74130
|
+
let headerRow = -1;
|
|
74131
|
+
for (let r6 = 1; r6 <= Math.min(HEADER_SCAN_ROWS, rowCount); r6++) {
|
|
74132
|
+
if (nonEmpty(r6) >= threshold && r6 < rowCount && nonEmpty(r6 + 1) >= 2) {
|
|
74133
|
+
headerRow = r6;
|
|
74134
|
+
break;
|
|
74135
|
+
}
|
|
74136
|
+
}
|
|
74137
|
+
if (headerRow < 0) return [];
|
|
74138
|
+
const cols = [];
|
|
74139
|
+
const seen = /* @__PURE__ */ new Set();
|
|
74140
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
74141
|
+
const hv = cellValue(ws.getCell(headerRow, c6).value);
|
|
74142
|
+
if (!isFilled(hv)) continue;
|
|
74143
|
+
const base = String(hv).replace(/\s+/g, " ").trim();
|
|
74144
|
+
if (!base) continue;
|
|
74145
|
+
let name = base;
|
|
74146
|
+
let i6 = 2;
|
|
74147
|
+
while (seen.has(name)) name = base + " " + String(i6++);
|
|
74148
|
+
seen.add(name);
|
|
74149
|
+
cols.push({ c: c6, name });
|
|
74150
|
+
}
|
|
74151
|
+
if (cols.length === 0) return [];
|
|
74152
|
+
const records = [];
|
|
74153
|
+
for (let r6 = headerRow + 1; r6 <= rowCount; r6++) {
|
|
74154
|
+
const row = {};
|
|
74155
|
+
let any = false;
|
|
74156
|
+
for (const { c: c6, name } of cols) {
|
|
74157
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
74158
|
+
if (isFilled(v2)) {
|
|
74159
|
+
row[name] = v2;
|
|
74160
|
+
any = true;
|
|
74161
|
+
}
|
|
74162
|
+
}
|
|
74163
|
+
if (!any) break;
|
|
74164
|
+
const first = cols[0] ? row[cols[0].name] : void 0;
|
|
74165
|
+
if (typeof first === "string" && /^total\b/i.test(first.trim())) continue;
|
|
74166
|
+
records.push(row);
|
|
74167
|
+
}
|
|
74168
|
+
return records;
|
|
74169
|
+
}
|
|
74170
|
+
var preambleCache = /* @__PURE__ */ new Map();
|
|
74171
|
+
function excelPreambleText(absPath) {
|
|
74172
|
+
return preambleCache.get((0, import_node_path40.resolve)(absPath)) ?? "";
|
|
74173
|
+
}
|
|
74174
|
+
function sheetPreamble(ws) {
|
|
74175
|
+
const lines = [];
|
|
74176
|
+
const rowCount = Math.min(10, ws.rowCount);
|
|
74177
|
+
const colCount = Math.min(8, ws.columnCount);
|
|
74178
|
+
for (let r6 = 1; r6 <= rowCount; r6++) {
|
|
74179
|
+
const cells = [];
|
|
74180
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
74181
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
74182
|
+
if (isFilled(v2)) cells.push(String(v2));
|
|
74183
|
+
}
|
|
74184
|
+
if (cells.length) lines.push(cells.join(" "));
|
|
74185
|
+
}
|
|
74186
|
+
return lines.join("\n");
|
|
74187
|
+
}
|
|
74188
|
+
async function excelToRecords(absPath) {
|
|
74189
|
+
let mod;
|
|
74190
|
+
try {
|
|
74191
|
+
mod = await import("exceljs");
|
|
74192
|
+
} catch {
|
|
74193
|
+
throw new Error(
|
|
74194
|
+
'Reading Excel files needs the "exceljs" package \u2014 install it with: npm install exceljs'
|
|
74195
|
+
);
|
|
74196
|
+
}
|
|
74197
|
+
const ExcelJS = mod.default ?? mod;
|
|
74198
|
+
const wb = new ExcelJS.Workbook();
|
|
74199
|
+
await wb.xlsx.readFile(absPath);
|
|
74200
|
+
const out = {};
|
|
74201
|
+
const preamble = [];
|
|
74202
|
+
const props = wb.properties;
|
|
74203
|
+
if (props?.title) preamble.push(props.title);
|
|
74204
|
+
for (const ws of wb.worksheets) {
|
|
74205
|
+
preamble.push(ws.name, sheetPreamble(ws));
|
|
74206
|
+
const records = sheetToRecords(ws);
|
|
74207
|
+
if (records.length > 0) out[ws.name] = records;
|
|
74208
|
+
}
|
|
74209
|
+
preambleCache.set((0, import_node_path40.resolve)(absPath), preamble.filter(Boolean).join("\n"));
|
|
74210
|
+
return out;
|
|
74211
|
+
}
|
|
74212
|
+
|
|
74213
|
+
// src/import/match.ts
|
|
74214
|
+
var BOOKKEEPING = /* @__PURE__ */ new Set(["id", "as_of", "content_key", "deleted_at"]);
|
|
74215
|
+
var MATCH_THRESHOLD = 0.6;
|
|
74216
|
+
function signature(columns) {
|
|
74217
|
+
const out = /* @__PURE__ */ new Set();
|
|
74218
|
+
for (const c6 of columns) {
|
|
74219
|
+
const n3 = normalizeName(c6);
|
|
74220
|
+
if (!n3 || BOOKKEEPING.has(n3) || n3.endsWith("_id")) continue;
|
|
74221
|
+
out.add(n3);
|
|
74222
|
+
}
|
|
74223
|
+
return out;
|
|
74224
|
+
}
|
|
74225
|
+
function containment(a6, b6) {
|
|
74226
|
+
if (a6.size === 0) return 0;
|
|
74227
|
+
let hit = 0;
|
|
74228
|
+
for (const c6 of a6) if (b6.has(c6)) hit++;
|
|
74229
|
+
return hit / a6.size;
|
|
74230
|
+
}
|
|
74231
|
+
function matchSchemaToExisting(existing, plan) {
|
|
74232
|
+
const ex = existing.map((t8) => ({ name: t8.name, sig: signature(t8.columns) }));
|
|
74233
|
+
const matches = [];
|
|
74234
|
+
const rename = {};
|
|
74235
|
+
for (const ent of plan.entities) {
|
|
74236
|
+
const sig = signature(ent.columns.map((c6) => c6.name));
|
|
74237
|
+
if (sig.size === 0) continue;
|
|
74238
|
+
let best = null;
|
|
74239
|
+
for (const t8 of ex) {
|
|
74240
|
+
if (normalizeName(t8.name) === normalizeName(ent.name)) {
|
|
74241
|
+
best = { name: t8.name, overlap: 1 };
|
|
74242
|
+
break;
|
|
74243
|
+
}
|
|
74244
|
+
const overlap = containment(sig, t8.sig);
|
|
74245
|
+
if (overlap > (best?.overlap ?? 0)) best = { name: t8.name, overlap };
|
|
74246
|
+
}
|
|
74247
|
+
if (best && best.overlap >= MATCH_THRESHOLD) {
|
|
74248
|
+
matches.push({ from: ent.name, to: best.name, overlap: best.overlap });
|
|
74249
|
+
if (best.name !== ent.name) rename[ent.name] = best.name;
|
|
74250
|
+
}
|
|
74251
|
+
}
|
|
74252
|
+
const totalEntities = plan.entities.length;
|
|
74253
|
+
const matchedCount = matches.length;
|
|
74254
|
+
const isKnownDocument = totalEntities > 0 && matchedCount >= Math.ceil(totalEntities / 2);
|
|
74255
|
+
return { matches, rename, matchedCount, totalEntities, isKnownDocument };
|
|
74256
|
+
}
|
|
74257
|
+
function renameEntities(plan, views, rename) {
|
|
74258
|
+
if (Object.keys(rename).length === 0) return { plan, views };
|
|
74259
|
+
const r6 = (n3) => rename[n3] ?? n3;
|
|
74260
|
+
return {
|
|
74261
|
+
plan: {
|
|
74262
|
+
...plan,
|
|
74263
|
+
entities: plan.entities.map((e6) => ({ ...e6, name: r6(e6.name) })),
|
|
74264
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.map(r6) })),
|
|
74265
|
+
linkages: plan.linkages.map((l4) => ({
|
|
74266
|
+
...l4,
|
|
74267
|
+
fromEntity: r6(l4.fromEntity),
|
|
74268
|
+
toEntity: r6(l4.toEntity),
|
|
74269
|
+
...l4.junction ? { junction: l4.junction } : {}
|
|
74270
|
+
}))
|
|
74271
|
+
},
|
|
74272
|
+
views: views.map((v2) => ({ ...v2, name: r6(v2.name), master: r6(v2.master) }))
|
|
74273
|
+
};
|
|
74274
|
+
}
|
|
74275
|
+
|
|
74276
|
+
// src/import/materialize.ts
|
|
74277
|
+
var import_node_crypto22 = require("crypto");
|
|
74278
|
+
var import_node_fs38 = require("fs");
|
|
74279
|
+
init_parser();
|
|
74280
|
+
init_normalize();
|
|
74281
|
+
|
|
74282
|
+
// src/import/asof.ts
|
|
74283
|
+
var MONTHS2 = {
|
|
74284
|
+
jan: 1,
|
|
74285
|
+
january: 1,
|
|
74286
|
+
feb: 2,
|
|
74287
|
+
february: 2,
|
|
74288
|
+
mar: 3,
|
|
74289
|
+
march: 3,
|
|
74290
|
+
apr: 4,
|
|
74291
|
+
april: 4,
|
|
74292
|
+
may: 5,
|
|
74293
|
+
jun: 6,
|
|
74294
|
+
june: 6,
|
|
74295
|
+
jul: 7,
|
|
74296
|
+
july: 7,
|
|
74297
|
+
aug: 8,
|
|
74298
|
+
august: 8,
|
|
74299
|
+
sep: 9,
|
|
74300
|
+
sept: 9,
|
|
74301
|
+
september: 9,
|
|
74302
|
+
oct: 10,
|
|
74303
|
+
october: 10,
|
|
74304
|
+
nov: 11,
|
|
74305
|
+
november: 11,
|
|
74306
|
+
dec: 12,
|
|
74307
|
+
december: 12
|
|
74308
|
+
};
|
|
74309
|
+
var ASOF_KEYWORDS = /\b(as[ -]?of|as at|period (?:end(?:ed|ing)?|of)|fye|fiscal year end(?:ed|ing)?|year[ -]?end(?:ed|ing)?|quarter[ -]?end(?:ed|ing)?|valuation date|report(?:ing)? date|effective date|dated)\b/i;
|
|
74310
|
+
function isoFrom(y2, m4, d6) {
|
|
74311
|
+
if (m4 < 1 || m4 > 12 || d6 < 1 || d6 > 31) return null;
|
|
74312
|
+
if (y2 < 2010 || y2 > 2099) return null;
|
|
74313
|
+
return `${String(y2)}-${String(m4).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
74314
|
+
}
|
|
74315
|
+
function findDates(text) {
|
|
74316
|
+
const hits = [];
|
|
74317
|
+
const push = (date2, match, index) => {
|
|
74318
|
+
if (date2) hits.push({ date: date2, match, index });
|
|
74319
|
+
};
|
|
74320
|
+
for (const m4 of text.matchAll(/(20\d{2})[-._/](\d{1,2})[-._/](\d{1,2})/g)) {
|
|
74321
|
+
push(isoFrom(Number(m4[1]), Number(m4[2]), Number(m4[3])), m4[0], m4.index);
|
|
74322
|
+
}
|
|
74323
|
+
for (const m4 of text.matchAll(/(\d{1,2})[-._/](\d{1,2})[-._/](\d{2,4})/g)) {
|
|
74324
|
+
let y2 = Number(m4[3]);
|
|
74325
|
+
if (y2 < 100) y2 += 2e3;
|
|
74326
|
+
push(isoFrom(y2, Number(m4[1]), Number(m4[2])), m4[0], m4.index);
|
|
74327
|
+
}
|
|
74328
|
+
for (const m4 of text.matchAll(/([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(20\d{2})/g)) {
|
|
74329
|
+
const mon = MONTHS2[(m4[1] ?? "").toLowerCase()];
|
|
74330
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[2])), m4[0], m4.index);
|
|
74331
|
+
}
|
|
74332
|
+
for (const m4 of text.matchAll(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(20\d{2})/g)) {
|
|
74333
|
+
const mon = MONTHS2[(m4[2] ?? "").toLowerCase()];
|
|
74334
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[1])), m4[0], m4.index);
|
|
74335
|
+
}
|
|
74336
|
+
return hits;
|
|
74337
|
+
}
|
|
74338
|
+
function parseCellDate(value) {
|
|
74339
|
+
if (value instanceof Date) {
|
|
74340
|
+
return isoFrom(value.getUTCFullYear(), value.getUTCMonth() + 1, value.getUTCDate());
|
|
74341
|
+
}
|
|
74342
|
+
if (typeof value === "string") return findDates(value)[0]?.date ?? null;
|
|
74343
|
+
return null;
|
|
74344
|
+
}
|
|
74345
|
+
function scanText(text, label) {
|
|
74346
|
+
if (!text) return [];
|
|
74347
|
+
const out = [];
|
|
74348
|
+
for (const hit of findDates(text)) {
|
|
74349
|
+
const before = text.slice(Math.max(0, hit.index - 40), hit.index);
|
|
74350
|
+
const keyworded = ASOF_KEYWORDS.test(before) || ASOF_KEYWORDS.test(hit.match);
|
|
74351
|
+
const snippet = text.slice(Math.max(0, hit.index - 24), hit.index + hit.match.length + 4).replace(/\s+/g, " ").trim();
|
|
74352
|
+
out.push({
|
|
74353
|
+
date: hit.date,
|
|
74354
|
+
source: "content",
|
|
74355
|
+
confidence: keyworded ? 0.95 : 0.7,
|
|
74356
|
+
evidence: `${label}: "${snippet}"`
|
|
74357
|
+
});
|
|
74358
|
+
}
|
|
74359
|
+
return out;
|
|
74360
|
+
}
|
|
74361
|
+
function scanFilename(fileName) {
|
|
74362
|
+
if (!fileName) return [];
|
|
74363
|
+
const base = fileName.replace(/\.[A-Za-z0-9]+$/, "");
|
|
74364
|
+
return findDates(base).map((hit, i6, all) => ({
|
|
74365
|
+
date: hit.date,
|
|
74366
|
+
source: "filename",
|
|
74367
|
+
confidence: i6 === all.length - 1 ? 0.6 : 0.45,
|
|
74368
|
+
evidence: `file name: "${hit.match}"`
|
|
74369
|
+
}));
|
|
74370
|
+
}
|
|
74371
|
+
function detectAsOfCandidates(inputs) {
|
|
74372
|
+
const all = [];
|
|
74373
|
+
for (const t8 of inputs.texts ?? []) all.push(...scanText(t8.text, t8.label));
|
|
74374
|
+
if (inputs.fileName) all.push(...scanFilename(inputs.fileName));
|
|
74375
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
74376
|
+
for (const c6 of all) {
|
|
74377
|
+
const prev = byDate.get(c6.date);
|
|
74378
|
+
if (!prev || c6.confidence > prev.confidence) byDate.set(c6.date, c6);
|
|
74379
|
+
}
|
|
74380
|
+
return [...byDate.values()].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
74381
|
+
}
|
|
74382
|
+
function detectAsOf(fileName) {
|
|
74383
|
+
return scanFilename(fileName)[0]?.date ?? null;
|
|
74384
|
+
}
|
|
74385
|
+
|
|
74386
|
+
// src/import/materialize.ts
|
|
74387
|
+
function coerce2(v2, type) {
|
|
74388
|
+
if (v2 === null || v2 === void 0 || v2 === "") return null;
|
|
74389
|
+
if (type === "boolean") return v2 === true || v2 === "true" || v2 === 1 ? 1 : 0;
|
|
74390
|
+
return v2;
|
|
74391
|
+
}
|
|
74392
|
+
function contentKey(record) {
|
|
74393
|
+
const parts = Object.keys(record).sort().map((k6) => k6 + "=" + JSON.stringify(record[k6] ?? null));
|
|
74394
|
+
return (0, import_node_crypto22.createHash)("sha256").update(parts.join("|")).digest("hex");
|
|
74395
|
+
}
|
|
74396
|
+
function persistTable(configPath, name, fields) {
|
|
74397
|
+
if (!configPath || !(0, import_node_fs38.existsSync)(configPath)) return;
|
|
74398
|
+
try {
|
|
74399
|
+
const doc = loadConfigDoc(configPath);
|
|
74400
|
+
doc.setIn(["entities", name], { fields, outputFile: name.toUpperCase() + ".md" });
|
|
74401
|
+
saveConfigDoc(configPath, doc);
|
|
74402
|
+
} catch {
|
|
74403
|
+
}
|
|
74404
|
+
}
|
|
74405
|
+
async function materializeImport(ctx, data, plan, views = [], opts = {}) {
|
|
74406
|
+
const { db, configPath } = ctx;
|
|
74407
|
+
const mode = opts.mode ?? "both";
|
|
74408
|
+
const doSchema = mode === "schema" || mode === "both";
|
|
74409
|
+
const doContents = mode === "contents" || mode === "both";
|
|
74410
|
+
const asOf = opts.asOf?.trim() ? opts.asOf.trim() : null;
|
|
74411
|
+
const asOfColumn = opts.asOfColumn?.trim() ? opts.asOfColumn.trim() : null;
|
|
74412
|
+
const dated = asOf !== null || asOfColumn !== null;
|
|
74413
|
+
const asOfSourceKey = (entity) => asOfColumn ? entity.columns.find((c6) => c6.name === asOfColumn)?.sourceKey ?? null : null;
|
|
74414
|
+
const rowAsOf = (entity, record) => {
|
|
74415
|
+
const sk = asOfSourceKey(entity);
|
|
74416
|
+
if (sk) {
|
|
74417
|
+
const d6 = parseCellDate(record[sk]);
|
|
74418
|
+
if (d6) return d6;
|
|
74419
|
+
}
|
|
74420
|
+
return asOf;
|
|
74421
|
+
};
|
|
74422
|
+
const recordKey = (entity, record) => {
|
|
74423
|
+
const a6 = rowAsOf(entity, record);
|
|
74424
|
+
return a6 ? contentKey({ ...record, __as_of: a6 }) : contentKey(record);
|
|
74425
|
+
};
|
|
74426
|
+
const scopedKey = (a6, keyVal) => (a6 ?? "") + "|" + normalizeText(keyVal);
|
|
74427
|
+
const report = async (p3) => {
|
|
74428
|
+
await opts.onProgress?.(p3);
|
|
74429
|
+
};
|
|
74430
|
+
const tablesCreated = [];
|
|
74431
|
+
const rowsByTable = {};
|
|
74432
|
+
const links = [];
|
|
74433
|
+
const viewResults = [];
|
|
74434
|
+
const byName = new Map(plan.entities.map((e6) => [e6.name, e6]));
|
|
74435
|
+
for (const entity of plan.entities) {
|
|
74436
|
+
const keyless = entity.naturalKey === null;
|
|
74437
|
+
const columns = { id: "TEXT PRIMARY KEY" };
|
|
74438
|
+
const fieldTypes = {};
|
|
74439
|
+
const cfgFields = { id: { type: "uuid", primaryKey: true } };
|
|
74440
|
+
for (const c6 of entity.columns) {
|
|
74441
|
+
columns[c6.name] = fieldToSqliteBaseType(c6.type);
|
|
74442
|
+
fieldTypes[c6.name] = c6.type;
|
|
74443
|
+
cfgFields[c6.name] = { type: c6.type };
|
|
74444
|
+
}
|
|
74445
|
+
const needsContentKey = keyless || dated;
|
|
74446
|
+
if (needsContentKey) {
|
|
74447
|
+
columns.content_key = "TEXT";
|
|
74448
|
+
cfgFields.content_key = { type: "text" };
|
|
74449
|
+
}
|
|
74450
|
+
if (dated) {
|
|
74451
|
+
columns.as_of = "TEXT";
|
|
74452
|
+
cfgFields.as_of = { type: "text" };
|
|
74453
|
+
}
|
|
74454
|
+
columns.deleted_at = "TEXT";
|
|
74455
|
+
cfgFields.deleted_at = { type: "text" };
|
|
74456
|
+
if (!db.getRegisteredTableNames().includes(entity.name)) tablesCreated.push(entity.name);
|
|
74457
|
+
await db.defineLate(entity.name, { columns, fieldTypes, primaryKey: "id" });
|
|
74458
|
+
persistTable(configPath, entity.name, cfgFields);
|
|
74459
|
+
await report({
|
|
74460
|
+
phase: "entities",
|
|
74461
|
+
table: entity.name,
|
|
74462
|
+
message: `Created table ${entity.name}`
|
|
74463
|
+
});
|
|
74464
|
+
if (doContents) {
|
|
74465
|
+
const records = sourceRecords(data, entity);
|
|
74466
|
+
const rows = records.map((r6) => {
|
|
74467
|
+
const row = {};
|
|
74468
|
+
for (const c6 of entity.columns) row[c6.name] = coerce2(r6[c6.sourceKey], c6.type);
|
|
74469
|
+
if (needsContentKey) row.content_key = recordKey(entity, r6);
|
|
74470
|
+
if (dated) row.as_of = rowAsOf(entity, r6);
|
|
74471
|
+
return row;
|
|
74472
|
+
});
|
|
74473
|
+
await db.seed({
|
|
74474
|
+
data: rows,
|
|
74475
|
+
table: entity.name,
|
|
74476
|
+
naturalKey: dated ? "content_key" : entity.naturalKey ?? "content_key"
|
|
74477
|
+
});
|
|
74478
|
+
const n3 = await db.count(entity.name);
|
|
74479
|
+
rowsByTable[entity.name] = n3;
|
|
74480
|
+
await report({
|
|
74481
|
+
phase: "entities",
|
|
74482
|
+
table: entity.name,
|
|
74483
|
+
count: n3,
|
|
74484
|
+
message: `Loaded ${String(n3)} rows into ${entity.name}`
|
|
74485
|
+
});
|
|
74486
|
+
}
|
|
74487
|
+
}
|
|
74488
|
+
for (const dim of plan.dimensions) {
|
|
74489
|
+
if (!db.getRegisteredTableNames().includes(dim.name)) tablesCreated.push(dim.name);
|
|
74490
|
+
await db.defineLate(dim.name, {
|
|
74491
|
+
columns: { id: "TEXT PRIMARY KEY", value: "TEXT", deleted_at: "TEXT" },
|
|
74492
|
+
fieldTypes: { value: "text" },
|
|
74493
|
+
primaryKey: "id"
|
|
74494
|
+
});
|
|
74495
|
+
persistTable(configPath, dim.name, {
|
|
74496
|
+
id: { type: "uuid", primaryKey: true },
|
|
74497
|
+
value: { type: "text" },
|
|
74498
|
+
deleted_at: { type: "text" }
|
|
74499
|
+
});
|
|
74500
|
+
if (doSchema) {
|
|
74501
|
+
const values = /* @__PURE__ */ new Map();
|
|
74502
|
+
for (const ename of dim.fromEntities) {
|
|
74503
|
+
const ent = byName.get(ename);
|
|
74504
|
+
if (!ent) continue;
|
|
74505
|
+
const records = sourceRecords(data, ent);
|
|
74506
|
+
const first = records[0];
|
|
74507
|
+
const srcKey = first ? Object.keys(first).find((k6) => normalizeName(k6) === dim.name) : void 0;
|
|
74508
|
+
if (!srcKey) continue;
|
|
74509
|
+
for (const r6 of records) {
|
|
74510
|
+
const v2 = r6[srcKey];
|
|
74511
|
+
if (typeof v2 !== "string" && typeof v2 !== "number") continue;
|
|
74512
|
+
const key = normalizeText(v2);
|
|
74513
|
+
if (key !== "" && !values.has(key)) values.set(key, String(v2));
|
|
74514
|
+
}
|
|
74515
|
+
}
|
|
74516
|
+
await db.seed({
|
|
74517
|
+
data: [...values.values()].map((value) => ({ value })),
|
|
74518
|
+
table: dim.name,
|
|
74519
|
+
naturalKey: "value"
|
|
74520
|
+
});
|
|
74521
|
+
const n3 = await db.count(dim.name);
|
|
74522
|
+
rowsByTable[dim.name] = n3;
|
|
74523
|
+
await report({
|
|
74524
|
+
phase: "dimensions",
|
|
74525
|
+
table: dim.name,
|
|
74526
|
+
count: n3,
|
|
74527
|
+
message: `Dimension ${dim.name}: ${String(n3)} values`
|
|
74528
|
+
});
|
|
74529
|
+
}
|
|
74530
|
+
}
|
|
74531
|
+
const idMapCache = /* @__PURE__ */ new Map();
|
|
74532
|
+
async function idMap(table, keyCol, datedTarget) {
|
|
74533
|
+
const cacheKey = table + ":" + keyCol + ":" + (datedTarget ? "D" : "");
|
|
74534
|
+
const cached = idMapCache.get(cacheKey);
|
|
74535
|
+
if (cached) return cached;
|
|
74536
|
+
const map = /* @__PURE__ */ new Map();
|
|
74537
|
+
for (const r6 of await db.query(table)) {
|
|
74538
|
+
const k6 = r6[keyCol];
|
|
74539
|
+
if (k6 === null || k6 === void 0) continue;
|
|
74540
|
+
const mapKey = datedTarget ? scopedKey(r6.as_of, k6) : normalizeText(k6);
|
|
74541
|
+
map.set(mapKey, String(r6.id));
|
|
74542
|
+
}
|
|
74543
|
+
idMapCache.set(cacheKey, map);
|
|
74544
|
+
return map;
|
|
74545
|
+
}
|
|
74546
|
+
for (const link of plan.linkages) {
|
|
74547
|
+
const from = byName.get(link.fromEntity);
|
|
74548
|
+
if (!from) continue;
|
|
74549
|
+
const jName = link.junction ?? `${link.fromEntity}_${link.toEntity}`;
|
|
74550
|
+
const fromFk = `${link.fromEntity}_id`;
|
|
74551
|
+
const toFk = `${link.toEntity}_id`;
|
|
74552
|
+
const jCols = {
|
|
74553
|
+
id: "TEXT PRIMARY KEY",
|
|
74554
|
+
[fromFk]: "TEXT",
|
|
74555
|
+
[toFk]: "TEXT"
|
|
74556
|
+
};
|
|
74557
|
+
const jCfg = {
|
|
74558
|
+
id: { type: "uuid", primaryKey: true },
|
|
74559
|
+
[fromFk]: { type: "uuid", ref: link.fromEntity },
|
|
74560
|
+
[toFk]: { type: "uuid", ref: link.toEntity }
|
|
74561
|
+
};
|
|
74562
|
+
if (dated) {
|
|
74563
|
+
jCols.as_of = "TEXT";
|
|
74564
|
+
jCfg.as_of = { type: "text" };
|
|
74565
|
+
}
|
|
74566
|
+
if (!db.getRegisteredTableNames().includes(jName)) tablesCreated.push(jName);
|
|
74567
|
+
await db.defineLate(jName, { columns: jCols, primaryKey: "id" });
|
|
74568
|
+
persistTable(configPath, jName, jCfg);
|
|
74569
|
+
if (!doContents) continue;
|
|
74570
|
+
const fromKeyCol = from.naturalKey ?? "content_key";
|
|
74571
|
+
const toIsEntity = byName.has(link.toEntity);
|
|
74572
|
+
const fromMap = await idMap(link.fromEntity, fromKeyCol, dated);
|
|
74573
|
+
const toMap = await idMap(link.toEntity, link.toKey, toIsEntity && dated);
|
|
74574
|
+
const seen = /* @__PURE__ */ new Set();
|
|
74575
|
+
for (const r6 of await db.query(jName)) {
|
|
74576
|
+
seen.add(String(r6[fromFk]) + "|" + String(r6[toFk]));
|
|
74577
|
+
}
|
|
74578
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
74579
|
+
let created = 0;
|
|
74580
|
+
for (const record of sourceRecords(data, from)) {
|
|
74581
|
+
const a6 = rowAsOf(from, record);
|
|
74582
|
+
const fromKeyVal = from.naturalKey === null ? recordKey(from, record) : record[from.naturalKeySource ?? ""];
|
|
74583
|
+
const fromId = fromMap.get(dated ? scopedKey(a6, fromKeyVal) : normalizeText(fromKeyVal));
|
|
74584
|
+
if (!fromId) continue;
|
|
74585
|
+
const raw = record[link.fromField];
|
|
74586
|
+
const refs = Array.isArray(raw) ? raw : [raw];
|
|
74587
|
+
for (const ref of refs) {
|
|
74588
|
+
if (ref === null || ref === void 0 || ref === "") continue;
|
|
74589
|
+
const toId = toMap.get(toIsEntity && dated ? scopedKey(a6, ref) : normalizeText(ref));
|
|
74590
|
+
if (!toId) {
|
|
74591
|
+
unresolved.add(normalizeText(ref));
|
|
74592
|
+
continue;
|
|
74593
|
+
}
|
|
74594
|
+
const edge = fromId + "|" + toId;
|
|
74595
|
+
if (seen.has(edge)) continue;
|
|
74596
|
+
seen.add(edge);
|
|
74597
|
+
await db.insert(
|
|
74598
|
+
jName,
|
|
74599
|
+
dated ? { [fromFk]: fromId, [toFk]: toId, as_of: a6 } : { [fromFk]: fromId, [toFk]: toId }
|
|
74600
|
+
);
|
|
74601
|
+
created++;
|
|
74602
|
+
}
|
|
74603
|
+
}
|
|
74604
|
+
rowsByTable[jName] = created;
|
|
74605
|
+
links.push({ junction: jName, created, unresolved: unresolved.size });
|
|
74606
|
+
await report({
|
|
74607
|
+
phase: "links",
|
|
74608
|
+
table: jName,
|
|
74609
|
+
count: created,
|
|
74610
|
+
message: `Linked ${String(created)} ${jName}`
|
|
74611
|
+
});
|
|
74612
|
+
}
|
|
74613
|
+
if (doSchema) {
|
|
74614
|
+
for (const v2 of views) {
|
|
74615
|
+
const filt = v2.filterValue.replace(/'/g, "''");
|
|
74616
|
+
await execSql(db, `DROP VIEW IF EXISTS "${v2.name}"`);
|
|
74617
|
+
await execSql(
|
|
74618
|
+
db,
|
|
74619
|
+
`CREATE VIEW "${v2.name}" AS SELECT * FROM "${v2.master}" WHERE "${v2.filterColumn}" = '${filt}'`
|
|
74620
|
+
);
|
|
74621
|
+
const cols = await db.introspectColumns(v2.name);
|
|
74622
|
+
await db.defineLate(v2.name, {
|
|
74623
|
+
columns: Object.fromEntries(cols.map((c6) => [c6, "TEXT"])),
|
|
74624
|
+
render: () => ""
|
|
74625
|
+
});
|
|
74626
|
+
if (!tablesCreated.includes(v2.name)) tablesCreated.push(v2.name);
|
|
74627
|
+
const rows = await db.count(v2.name);
|
|
74628
|
+
rowsByTable[v2.name] = rows;
|
|
74629
|
+
viewResults.push({ name: v2.name, master: v2.master, rows });
|
|
74630
|
+
await report({
|
|
74631
|
+
phase: "views",
|
|
74632
|
+
table: v2.name,
|
|
74633
|
+
count: rows,
|
|
74634
|
+
message: `View ${v2.name}: ${String(rows)} rows`
|
|
74635
|
+
});
|
|
74636
|
+
}
|
|
74637
|
+
}
|
|
74638
|
+
await report({ phase: "done", message: "Import complete" });
|
|
74639
|
+
return { mode, asOf, asOfColumn, tablesCreated, rowsByTable, links, views: viewResults };
|
|
74640
|
+
}
|
|
74641
|
+
|
|
74642
|
+
// src/gui/import-auto.ts
|
|
74643
|
+
init_native_entities();
|
|
74644
|
+
|
|
74645
|
+
// src/gui/import-detect.ts
|
|
74646
|
+
var import_node_path41 = require("path");
|
|
74647
|
+
|
|
74648
|
+
// src/gui/ai/asof-llm.ts
|
|
74649
|
+
init_assistant_routes();
|
|
74650
|
+
init_chat();
|
|
74651
|
+
var MAX_CHARS = 6e3;
|
|
74652
|
+
var SYSTEM = 'You extract the single "as of" / report / snapshot / period-end date from the text of a data file (a financial statement, track record, export, etc.). Reply with ONLY that date as ISO YYYY-MM-DD, or the exact word NONE if the text has no such date. Output nothing else \u2014 no prose, no quotes.';
|
|
74653
|
+
function parseLlmDate(reply) {
|
|
74654
|
+
if (!reply) return null;
|
|
74655
|
+
const m4 = /(20\d{2})-(\d{2})-(\d{2})/.exec(reply);
|
|
74656
|
+
if (!m4) return null;
|
|
74657
|
+
const y2 = Number(m4[1]);
|
|
74658
|
+
const mo = Number(m4[2]);
|
|
74659
|
+
const d6 = Number(m4[3]);
|
|
74660
|
+
if (mo < 1 || mo > 12 || d6 < 1 || d6 > 31 || y2 < 2010 || y2 > 2099) return null;
|
|
74661
|
+
return `${String(y2)}-${String(mo).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
74662
|
+
}
|
|
74663
|
+
async function asOfFromLlm(db, text) {
|
|
74664
|
+
const trimmed = text.trim();
|
|
74665
|
+
if (!trimmed) return null;
|
|
74666
|
+
try {
|
|
74667
|
+
const auth = await resolveClaudeAuth(db);
|
|
74668
|
+
if (!auth) return null;
|
|
74669
|
+
const client = createAnthropicClient(auth);
|
|
74670
|
+
const result = await client.runTurn({
|
|
74671
|
+
model: DEFAULT_MODEL2,
|
|
74672
|
+
system: SYSTEM,
|
|
74673
|
+
temperature: 0,
|
|
74674
|
+
tools: [],
|
|
74675
|
+
messages: [{ role: "user", content: `File text:
|
|
74676
|
+
${trimmed.slice(0, MAX_CHARS)}` }],
|
|
74677
|
+
onText: () => {
|
|
74678
|
+
}
|
|
74679
|
+
});
|
|
74680
|
+
const date2 = parseLlmDate(result.text);
|
|
74681
|
+
return date2 ? { date: date2, source: "llm", confidence: 0.85, evidence: "Claude read the file" } : null;
|
|
74682
|
+
} catch (e6) {
|
|
74683
|
+
console.warn("[import] as-of LLM fallback failed:", e6.message);
|
|
74684
|
+
return null;
|
|
74685
|
+
}
|
|
74686
|
+
}
|
|
74687
|
+
|
|
74688
|
+
// src/gui/import-detect.ts
|
|
74689
|
+
async function detectImportAsOf(db, data, opts = {}) {
|
|
74690
|
+
const fileName = opts.fileName ?? (opts.abs ? (0, import_node_path41.basename)(opts.abs).replace(/^[0-9a-f]{8}-/, "") : "");
|
|
74691
|
+
const texts = [];
|
|
74692
|
+
for (const [k6, v2] of Object.entries(data)) {
|
|
74693
|
+
if (!Array.isArray(v2)) texts.push({ label: "data", text: `${k6}: ${JSON.stringify(v2)}` });
|
|
74694
|
+
}
|
|
74695
|
+
if (opts.abs && /\.xlsx?$/i.test(opts.abs)) {
|
|
74696
|
+
const pre = excelPreambleText(opts.abs);
|
|
74697
|
+
if (pre) texts.push({ label: "title", text: pre });
|
|
74698
|
+
}
|
|
74699
|
+
let candidates = detectAsOfCandidates({ fileName, texts });
|
|
74700
|
+
if (!candidates[0] || candidates[0].confidence < 0.7) {
|
|
74701
|
+
const llm = await asOfFromLlm(db, texts.map((t8) => t8.text).join("\n"));
|
|
74702
|
+
if (llm) candidates = [...candidates, llm].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
74703
|
+
}
|
|
74704
|
+
return candidates;
|
|
74705
|
+
}
|
|
74706
|
+
|
|
74707
|
+
// src/import/asof-columns.ts
|
|
74708
|
+
var STRONG_NAME = /(as[_ -]?of|as[_ -]?at|report(?:ing)?[_ -]?date|valuation[_ -]?date|effective[_ -]?date|period[_ -]?end|snapshot[_ -]?date|statement[_ -]?date|fye)/i;
|
|
74709
|
+
var WEAK_NAME = /(^|_)(date|period|quarter|asof)($|_)/i;
|
|
74710
|
+
function detectAsOfColumns(data, plan) {
|
|
74711
|
+
const out = [];
|
|
74712
|
+
for (const entity of plan.entities) {
|
|
74713
|
+
const records = sourceRecords(data, entity);
|
|
74714
|
+
if (records.length < 2) continue;
|
|
74715
|
+
for (const col of entity.columns) {
|
|
74716
|
+
const strong = STRONG_NAME.test(col.name);
|
|
74717
|
+
const weak = WEAK_NAME.test(col.name);
|
|
74718
|
+
if (!strong && !weak) continue;
|
|
74719
|
+
const vals = records.map((r6) => r6[col.sourceKey]).filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
74720
|
+
if (vals.length < Math.max(3, Math.floor(records.length * 0.5))) continue;
|
|
74721
|
+
const dates = vals.map(parseCellDate).filter((d6) => d6 !== null);
|
|
74722
|
+
if (dates.length / vals.length < 0.8) continue;
|
|
74723
|
+
const distinctDates = new Set(dates).size;
|
|
74724
|
+
const typed = col.type === "date" || col.type === "datetime";
|
|
74725
|
+
let confidence = strong ? 0.9 : 0.6;
|
|
74726
|
+
if (typed) confidence += 0.03;
|
|
74727
|
+
if (distinctDates > 1) confidence += 0.04;
|
|
74728
|
+
out.push({
|
|
74729
|
+
entity: entity.name,
|
|
74730
|
+
column: col.name,
|
|
74731
|
+
confidence: Math.min(confidence, 0.97),
|
|
74732
|
+
distinctDates,
|
|
74733
|
+
evidence: `column "${col.name}" \u2014 ${String(distinctDates)} distinct date${distinctDates === 1 ? "" : "s"} across ${String(vals.length)} rows`
|
|
74734
|
+
});
|
|
74735
|
+
}
|
|
74736
|
+
}
|
|
74737
|
+
return out.sort((a6, b6) => b6.confidence - a6.confidence);
|
|
74738
|
+
}
|
|
74739
|
+
|
|
74740
|
+
// src/gui/import-auto.ts
|
|
74741
|
+
function existingDataTables(db) {
|
|
74742
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
74743
|
+
const out = [];
|
|
74744
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
74745
|
+
if (native.has(t8)) continue;
|
|
74746
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
74747
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
74748
|
+
}
|
|
74749
|
+
return out;
|
|
74750
|
+
}
|
|
74751
|
+
async function readStructured(abs, name) {
|
|
74752
|
+
if (/\.xlsx?$/i.test(name)) return excelToRecords(abs);
|
|
74753
|
+
return JSON.parse((0, import_node_fs39.readFileSync)(abs, "utf8"));
|
|
74754
|
+
}
|
|
74755
|
+
async function autoImportStructured(db, configPath, abs, name) {
|
|
74756
|
+
if (!/\.(xlsx?|json)$/i.test(name)) return null;
|
|
74757
|
+
let data;
|
|
74758
|
+
try {
|
|
74759
|
+
data = await readStructured(abs, name);
|
|
74760
|
+
} catch {
|
|
74761
|
+
return null;
|
|
74762
|
+
}
|
|
74763
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
74764
|
+
inferSchema(data),
|
|
74765
|
+
data
|
|
74766
|
+
);
|
|
74767
|
+
if (inferredPlan.entities.length === 0) return null;
|
|
74768
|
+
const schemaMatch = matchSchemaToExisting(existingDataTables(db), inferredPlan);
|
|
74769
|
+
const asOfCandidates = await detectImportAsOf(db, data, { abs, fileName: name });
|
|
74770
|
+
const asOf = asOfCandidates[0]?.date ?? null;
|
|
74771
|
+
const asOfColumns = detectAsOfColumns(data, inferredPlan);
|
|
74772
|
+
const proposal = {
|
|
74773
|
+
plan: inferredPlan,
|
|
74774
|
+
views: inferredViews,
|
|
74775
|
+
asOfCandidates,
|
|
74776
|
+
asOfColumns,
|
|
74777
|
+
schemaMatch,
|
|
74778
|
+
matchedCount: schemaMatch.matchedCount,
|
|
74779
|
+
totalEntities: schemaMatch.totalEntities,
|
|
74780
|
+
tables: [],
|
|
74781
|
+
rows: 0
|
|
74782
|
+
};
|
|
74783
|
+
if (!schemaMatch.isKnownDocument) {
|
|
74784
|
+
return { imported: false, reason: "new-dataset", asOf, ...proposal };
|
|
74785
|
+
}
|
|
74786
|
+
if (!asOf) {
|
|
74787
|
+
return { imported: false, reason: "needs-confirm", asOf: null, ...proposal };
|
|
74788
|
+
}
|
|
74789
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, schemaMatch.rename);
|
|
74790
|
+
const result = await materializeImport({ db, configPath }, data, plan, views, { asOf });
|
|
74791
|
+
const rows = Object.values(result.rowsByTable).reduce((a6, b6) => a6 + b6, 0);
|
|
74792
|
+
return {
|
|
74793
|
+
imported: true,
|
|
74794
|
+
asOf,
|
|
74795
|
+
matchedCount: schemaMatch.matchedCount,
|
|
74796
|
+
totalEntities: schemaMatch.totalEntities,
|
|
74797
|
+
tables: Object.keys(result.rowsByTable),
|
|
74798
|
+
rows
|
|
74799
|
+
};
|
|
74800
|
+
}
|
|
74801
|
+
|
|
74802
|
+
// src/gui/ingest-routes.ts
|
|
73207
74803
|
var MIME_BY_EXT = {
|
|
73208
74804
|
".pdf": "application/pdf",
|
|
73209
74805
|
".png": "image/png",
|
|
@@ -73230,7 +74826,7 @@ var MIME_BY_EXT = {
|
|
|
73230
74826
|
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
73231
74827
|
};
|
|
73232
74828
|
function mimeFor(name) {
|
|
73233
|
-
return MIME_BY_EXT[(0,
|
|
74829
|
+
return MIME_BY_EXT[(0, import_node_path42.extname)(name).toLowerCase()] ?? "application/octet-stream";
|
|
73234
74830
|
}
|
|
73235
74831
|
var RETAINABLE_DOC_MIMES = /* @__PURE__ */ new Set([
|
|
73236
74832
|
"application/pdf",
|
|
@@ -73339,7 +74935,6 @@ function looksLikeUrl(s2) {
|
|
|
73339
74935
|
const t8 = s2.trim();
|
|
73340
74936
|
return /^https?:\/\/\S+$/i.test(t8) && !/\s/.test(t8);
|
|
73341
74937
|
}
|
|
73342
|
-
var MAX_INGEST_BYTES = 5e7;
|
|
73343
74938
|
function readBuffer2(req, maxBytes = MAX_INGEST_BYTES) {
|
|
73344
74939
|
return new Promise((resolve_, reject) => {
|
|
73345
74940
|
const chunks = [];
|
|
@@ -73398,12 +74993,18 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73398
74993
|
sendJson(res, { error: "empty upload" }, 400);
|
|
73399
74994
|
return true;
|
|
73400
74995
|
}
|
|
73401
|
-
const tmp = (0,
|
|
74996
|
+
const tmp = (0, import_node_path42.join)((0, import_node_os9.tmpdir)(), `lattice-ingest-${crypto.randomUUID()}${(0, import_node_path42.extname)(name2)}`);
|
|
73402
74997
|
let result;
|
|
73403
74998
|
let blob = null;
|
|
74999
|
+
let autoImport = null;
|
|
73404
75000
|
try {
|
|
73405
75001
|
await (0, import_promises12.writeFile)(tmp, buf);
|
|
73406
75002
|
result = await extractSource(ctx.db, tmp, mime2, name2);
|
|
75003
|
+
try {
|
|
75004
|
+
autoImport = await autoImportStructured(ctx.db, ctx.configPath ?? null, tmp, name2);
|
|
75005
|
+
} catch (e6) {
|
|
75006
|
+
console.warn("[ingest] auto-import skipped:", e6.message);
|
|
75007
|
+
}
|
|
73407
75008
|
if (ctx.latticeRoot && !realPath && shouldRetainUploadBlob(mime2, name2)) {
|
|
73408
75009
|
try {
|
|
73409
75010
|
const meta2 = await attachBlob(tmp, ctx.latticeRoot);
|
|
@@ -73419,7 +75020,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73419
75020
|
let s3Status = null;
|
|
73420
75021
|
const s3cfg = resolveActiveS3Config(ctx.configPath);
|
|
73421
75022
|
if (s3cfg) {
|
|
73422
|
-
const sha256 = blob?.sha256 ?? (0,
|
|
75023
|
+
const sha256 = blob?.sha256 ?? (0, import_node_crypto23.createHash)("sha256").update(buf).digest("hex");
|
|
73423
75024
|
const key = s3Key(s3cfg.prefix, sha256);
|
|
73424
75025
|
try {
|
|
73425
75026
|
const store = await createS3Store(s3cfg);
|
|
@@ -73442,7 +75043,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73442
75043
|
}
|
|
73443
75044
|
}
|
|
73444
75045
|
const fileId = crypto.randomUUID();
|
|
73445
|
-
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ?? (0,
|
|
75046
|
+
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ?? (0, import_node_crypto23.createHash)("sha256").update(buf).digest("hex");
|
|
73446
75047
|
const uploadRow = {
|
|
73447
75048
|
id: fileId,
|
|
73448
75049
|
...fileIdentity(name2, fileId),
|
|
@@ -73480,6 +75081,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73480
75081
|
},
|
|
73481
75082
|
forcePrivate2 ? "private" : void 0
|
|
73482
75083
|
);
|
|
75084
|
+
if (autoImport?.reason) autoImport.fileId = id2;
|
|
73483
75085
|
try {
|
|
73484
75086
|
const dedupCtx = {
|
|
73485
75087
|
db: ctx.db,
|
|
@@ -73507,6 +75109,15 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73507
75109
|
e6 instanceof Error ? e6.message : String(e6)
|
|
73508
75110
|
);
|
|
73509
75111
|
}
|
|
75112
|
+
if (autoImport?.imported) {
|
|
75113
|
+
ctx.feed.publish({
|
|
75114
|
+
table: autoImport.tables[0] ?? "files",
|
|
75115
|
+
op: "insert",
|
|
75116
|
+
rowId: null,
|
|
75117
|
+
source: "system",
|
|
75118
|
+
summary: `Imported the ${autoImport.asOf ?? ""} snapshot of "${name2}" \u2014 ${String(autoImport.rows)} rows across ${String(autoImport.tables.length)} tables`
|
|
75119
|
+
});
|
|
75120
|
+
}
|
|
73510
75121
|
let suggestedLinks = [];
|
|
73511
75122
|
if (!result.skip) {
|
|
73512
75123
|
const links = await enrichOrFail(mctx, ctx.db, id2, result.text, name2, ctx, res, forcePrivate2);
|
|
@@ -73519,6 +75130,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73519
75130
|
id: id2,
|
|
73520
75131
|
extraction_status: result.skip ? "skipped" : "extracted",
|
|
73521
75132
|
suggestedLinks,
|
|
75133
|
+
...autoImport ? { autoImport } : {},
|
|
73522
75134
|
// Present only when S3 is enabled for this workspace. 'failed' tells the
|
|
73523
75135
|
// uploader the bytes did NOT reach the shared bucket — other members would
|
|
73524
75136
|
// 404 until it's re-uploaded — so the GUI can warn rather than imply a
|
|
@@ -73604,10 +75216,10 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73604
75216
|
sendJson(res, { error: "path is required" }, 400);
|
|
73605
75217
|
return true;
|
|
73606
75218
|
}
|
|
73607
|
-
const abs = (0,
|
|
75219
|
+
const abs = (0, import_node_path42.resolve)(rawPath);
|
|
73608
75220
|
let size = 0;
|
|
73609
75221
|
try {
|
|
73610
|
-
const st = (0,
|
|
75222
|
+
const st = (0, import_node_fs40.statSync)(abs);
|
|
73611
75223
|
if (!st.isFile()) {
|
|
73612
75224
|
sendJson(res, { error: "path is not a file" }, 400);
|
|
73613
75225
|
return true;
|
|
@@ -73621,7 +75233,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73621
75233
|
sendJson(res, { error: "file too large" }, 413);
|
|
73622
75234
|
return true;
|
|
73623
75235
|
}
|
|
73624
|
-
const name = (0,
|
|
75236
|
+
const name = (0, import_node_path42.basename)(abs);
|
|
73625
75237
|
const mime = mimeFor(name);
|
|
73626
75238
|
const localFileId = crypto.randomUUID();
|
|
73627
75239
|
const localRow = {
|
|
@@ -73687,6 +75299,146 @@ ${err.stack ?? ""}`
|
|
|
73687
75299
|
return true;
|
|
73688
75300
|
}
|
|
73689
75301
|
|
|
75302
|
+
// src/gui/import-routes.ts
|
|
75303
|
+
var import_node_fs41 = require("fs");
|
|
75304
|
+
var import_node_path43 = require("path");
|
|
75305
|
+
init_adapter();
|
|
75306
|
+
init_http2();
|
|
75307
|
+
init_native_entities();
|
|
75308
|
+
function badRequest(message) {
|
|
75309
|
+
const e6 = new Error(message);
|
|
75310
|
+
e6.statusCode = 400;
|
|
75311
|
+
return e6;
|
|
75312
|
+
}
|
|
75313
|
+
function localPathOf2(row, latticeRoot) {
|
|
75314
|
+
if (row.ref_kind === "local_ref" && row.ref_uri) return row.ref_uri;
|
|
75315
|
+
if ((row.ref_kind === "blob" || row.ref_kind === "cloud_ref") && row.blob_path) {
|
|
75316
|
+
return (0, import_node_path43.isAbsolute)(row.blob_path) ? row.blob_path : latticeRoot ? (0, import_node_path43.join)(latticeRoot, row.blob_path) : null;
|
|
75317
|
+
}
|
|
75318
|
+
return null;
|
|
75319
|
+
}
|
|
75320
|
+
function existingDataTables2(db) {
|
|
75321
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
75322
|
+
const out = [];
|
|
75323
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
75324
|
+
if (native.has(t8)) continue;
|
|
75325
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
75326
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
75327
|
+
}
|
|
75328
|
+
return out;
|
|
75329
|
+
}
|
|
75330
|
+
async function readImportSourceFromFile(db, fileId, latticeRoot) {
|
|
75331
|
+
const row = await getAsyncOrSync(
|
|
75332
|
+
db.adapter,
|
|
75333
|
+
`SELECT "id","original_name","mime","ref_kind","ref_uri","blob_path"
|
|
75334
|
+
FROM "files" WHERE "id" = ? AND "deleted_at" IS NULL LIMIT 1`,
|
|
75335
|
+
[fileId]
|
|
75336
|
+
);
|
|
75337
|
+
if (!row) throw badRequest("Unknown import file: " + fileId);
|
|
75338
|
+
const path2 = localPathOf2(row, latticeRoot);
|
|
75339
|
+
if (!path2 || !(0, import_node_fs41.existsSync)(path2)) {
|
|
75340
|
+
throw badRequest("The import file\u2019s bytes are not available locally.");
|
|
75341
|
+
}
|
|
75342
|
+
const sizeBytes = (0, import_node_fs41.statSync)(path2).size;
|
|
75343
|
+
if (sizeBytes > MAX_INGEST_BYTES) {
|
|
75344
|
+
throw badRequest(
|
|
75345
|
+
`The import file is too large (${String(Math.round(sizeBytes / 1e6))} MB); the limit is ${String(Math.round(MAX_INGEST_BYTES / 1e6))} MB.`
|
|
75346
|
+
);
|
|
75347
|
+
}
|
|
75348
|
+
const name = row.original_name ?? "";
|
|
75349
|
+
const mime = row.mime ?? "";
|
|
75350
|
+
if (/\.xlsx?$/i.test(name) || mime.includes("spreadsheet") || mime.includes("excel")) {
|
|
75351
|
+
return excelToRecords(path2);
|
|
75352
|
+
}
|
|
75353
|
+
let parsed;
|
|
75354
|
+
try {
|
|
75355
|
+
parsed = JSON.parse((0, import_node_fs41.readFileSync)(path2, "utf8"));
|
|
75356
|
+
} catch {
|
|
75357
|
+
throw badRequest("The import file is not valid JSON.");
|
|
75358
|
+
}
|
|
75359
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
75360
|
+
throw badRequest("Expected a JSON object whose keys are record arrays.");
|
|
75361
|
+
}
|
|
75362
|
+
return parsed;
|
|
75363
|
+
}
|
|
75364
|
+
async function dispatchImportRoute(req, res, deps) {
|
|
75365
|
+
const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
|
|
75366
|
+
if (req.method !== "POST" || pathname !== "/api/import/apply") return false;
|
|
75367
|
+
const body = await readJson(req).catch(() => ({}));
|
|
75368
|
+
const fileId = typeof body.fileId === "string" ? body.fileId : "";
|
|
75369
|
+
const mode = body.mode === "schema" || body.mode === "contents" ? body.mode : "both";
|
|
75370
|
+
const asOf = typeof body.asOf === "string" && /^\d{4}-\d{2}-\d{2}$/.test(body.asOf.trim()) ? body.asOf.trim() : null;
|
|
75371
|
+
const asOfColumn = typeof body.asOfColumn === "string" && body.asOfColumn.trim() ? body.asOfColumn.trim() : null;
|
|
75372
|
+
if (!fileId) {
|
|
75373
|
+
sendJson(res, { error: "fileId is required" }, 400);
|
|
75374
|
+
return true;
|
|
75375
|
+
}
|
|
75376
|
+
res.writeHead(200, {
|
|
75377
|
+
"content-type": "application/x-ndjson; charset=utf-8",
|
|
75378
|
+
"cache-control": "no-store"
|
|
75379
|
+
});
|
|
75380
|
+
const emit = (p3) => {
|
|
75381
|
+
res.write(JSON.stringify(p3) + "\n");
|
|
75382
|
+
};
|
|
75383
|
+
try {
|
|
75384
|
+
emit({ phase: "parse", message: "Reading source\u2026" });
|
|
75385
|
+
const data = await readImportSourceFromFile(deps.db, fileId, deps.latticeRoot);
|
|
75386
|
+
emit({ phase: "infer", message: "Analyzing schema\u2026" });
|
|
75387
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
75388
|
+
inferSchema(data),
|
|
75389
|
+
data
|
|
75390
|
+
);
|
|
75391
|
+
emit({
|
|
75392
|
+
phase: "infer",
|
|
75393
|
+
message: `Found ${String(inferredPlan.entities.length)} entities, ${String(inferredPlan.dimensions.length)} dimensions, ${String(inferredPlan.linkages.length)} links`
|
|
75394
|
+
});
|
|
75395
|
+
const match = matchSchemaToExisting(existingDataTables2(deps.db), inferredPlan);
|
|
75396
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, match.rename);
|
|
75397
|
+
if (views.length > 0) {
|
|
75398
|
+
emit({
|
|
75399
|
+
phase: "detect",
|
|
75400
|
+
message: `Detected ${String(views.length)} reconstructable views (no duplicated rows)`
|
|
75401
|
+
});
|
|
75402
|
+
}
|
|
75403
|
+
if (match.isKnownDocument) {
|
|
75404
|
+
emit({
|
|
75405
|
+
phase: "detect",
|
|
75406
|
+
message: `Recognized as a new period of an existing document \u2014 ${String(match.matchedCount)} of ${String(match.totalEntities)} tables matched`
|
|
75407
|
+
});
|
|
75408
|
+
}
|
|
75409
|
+
if (asOfColumn) {
|
|
75410
|
+
emit({ phase: "infer", message: `Dating each row by its "${asOfColumn}" column` });
|
|
75411
|
+
} else if (asOf) {
|
|
75412
|
+
emit({ phase: "infer", message: `Importing as a snapshot dated ${asOf}` });
|
|
75413
|
+
}
|
|
75414
|
+
const result = await materializeImport(
|
|
75415
|
+
{ db: deps.db, configPath: deps.configPath },
|
|
75416
|
+
data,
|
|
75417
|
+
plan,
|
|
75418
|
+
views,
|
|
75419
|
+
{
|
|
75420
|
+
mode,
|
|
75421
|
+
asOf,
|
|
75422
|
+
asOfColumn,
|
|
75423
|
+
onProgress: async (p3) => {
|
|
75424
|
+
emit({ ...p3 });
|
|
75425
|
+
await new Promise((r6) => setImmediate(r6));
|
|
75426
|
+
}
|
|
75427
|
+
}
|
|
75428
|
+
);
|
|
75429
|
+
for (const t8 of result.tablesCreated) {
|
|
75430
|
+
deps.validTables.add(t8);
|
|
75431
|
+
const cols = deps.db.getRegisteredColumns(t8);
|
|
75432
|
+
if (cols && "deleted_at" in cols) deps.softDeletable.add(t8);
|
|
75433
|
+
}
|
|
75434
|
+
emit({ phase: "done", ok: true, result });
|
|
75435
|
+
} catch (e6) {
|
|
75436
|
+
emit({ phase: "error", message: e6.message });
|
|
75437
|
+
}
|
|
75438
|
+
res.end();
|
|
75439
|
+
return true;
|
|
75440
|
+
}
|
|
75441
|
+
|
|
73690
75442
|
// src/gui/read-routes.ts
|
|
73691
75443
|
init_http2();
|
|
73692
75444
|
init_data();
|
|
@@ -73975,7 +75727,13 @@ async function handleReadRoutes(req, res, ctx, deps) {
|
|
|
73975
75727
|
return true;
|
|
73976
75728
|
}
|
|
73977
75729
|
if (method === "GET" && pathname === "/api/history") {
|
|
73978
|
-
const
|
|
75730
|
+
const limitRaw = url.searchParams.get("limit");
|
|
75731
|
+
const parsedLimit = parsePageParam(limitRaw, "limit");
|
|
75732
|
+
if (parsedLimit === "invalid") {
|
|
75733
|
+
sendJson(res, { error: "limit must be a non-negative integer" }, 400);
|
|
75734
|
+
return true;
|
|
75735
|
+
}
|
|
75736
|
+
const limit = limitRaw === null ? 200 : parsedLimit;
|
|
73979
75737
|
const filterTable = url.searchParams.get("table");
|
|
73980
75738
|
const raw = await active.db.query("_lattice_gui_audit", { limit });
|
|
73981
75739
|
let entries = raw.map(parseAudit).sort((a6, b6) => b6.ts.localeCompare(a6.ts));
|
|
@@ -75018,18 +76776,18 @@ async function handleHistoryRoutes(req, res, ctx, deps) {
|
|
|
75018
76776
|
}
|
|
75019
76777
|
|
|
75020
76778
|
// src/gui/workspaces-routes.ts
|
|
75021
|
-
var
|
|
75022
|
-
var
|
|
76779
|
+
var import_node_path44 = require("path");
|
|
76780
|
+
var import_node_fs42 = require("fs");
|
|
75023
76781
|
init_http2();
|
|
75024
76782
|
init_workspace();
|
|
75025
76783
|
init_lattice_root();
|
|
75026
76784
|
init_user_config();
|
|
75027
76785
|
function cleanupWorkspaceFiles(root6, ws) {
|
|
75028
76786
|
if (!ws.configPath && ws.kind === "local") {
|
|
75029
|
-
(0,
|
|
76787
|
+
(0, import_node_fs42.rmSync)(workspaceDir(root6, ws.dir), { recursive: true, force: true });
|
|
75030
76788
|
} else if (ws.kind === "cloud") {
|
|
75031
|
-
if (ws.configPath && (0,
|
|
75032
|
-
(0,
|
|
76789
|
+
if (ws.configPath && (0, import_node_fs42.existsSync)(ws.configPath)) {
|
|
76790
|
+
(0, import_node_fs42.rmSync)(ws.configPath, { force: true });
|
|
75033
76791
|
}
|
|
75034
76792
|
const labelMatch = /^\$\{LATTICE_DB:([A-Za-z0-9._-]+)\}$/.exec(ws.db.trim());
|
|
75035
76793
|
const label = labelMatch?.[1];
|
|
@@ -75182,7 +76940,7 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
75182
76940
|
return true;
|
|
75183
76941
|
}
|
|
75184
76942
|
const wsPaths = resolveWorkspacePaths(latticeRoot, ws);
|
|
75185
|
-
const isActive = (0,
|
|
76943
|
+
const isActive = (0, import_node_path44.resolve)(active.configPath) === (0, import_node_path44.resolve)(wsPaths.configPath);
|
|
75186
76944
|
let switchedTo = null;
|
|
75187
76945
|
if (isActive) {
|
|
75188
76946
|
const fallback = listWorkspaces(latticeRoot).find((w2) => w2.id !== ws.id);
|
|
@@ -75230,34 +76988,34 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
75230
76988
|
}
|
|
75231
76989
|
|
|
75232
76990
|
// src/gui/databases-routes.ts
|
|
75233
|
-
var
|
|
75234
|
-
var
|
|
76991
|
+
var import_node_path46 = require("path");
|
|
76992
|
+
var import_node_fs44 = require("fs");
|
|
75235
76993
|
init_http2();
|
|
75236
76994
|
init_parser();
|
|
75237
76995
|
|
|
75238
76996
|
// src/gui/config-paths.ts
|
|
75239
|
-
var
|
|
75240
|
-
var
|
|
76997
|
+
var import_node_path45 = require("path");
|
|
76998
|
+
var import_node_fs43 = require("fs");
|
|
75241
76999
|
var import_yaml10 = require("yaml");
|
|
75242
77000
|
init_parser();
|
|
75243
77001
|
function resolveOutputDirForConfig(configPath) {
|
|
75244
|
-
const base = (0,
|
|
77002
|
+
const base = (0, import_node_path45.dirname)((0, import_node_path45.resolve)(configPath));
|
|
75245
77003
|
for (const dir of ["context", ".", "generated"]) {
|
|
75246
|
-
const abs = (0,
|
|
75247
|
-
if ((0,
|
|
77004
|
+
const abs = (0, import_node_path45.resolve)(base, dir);
|
|
77005
|
+
if ((0, import_node_fs43.existsSync)((0, import_node_path45.join)(abs, ".lattice", "manifest.json"))) return abs;
|
|
75248
77006
|
}
|
|
75249
|
-
return (0,
|
|
77007
|
+
return (0, import_node_path45.resolve)(base, "context");
|
|
75250
77008
|
}
|
|
75251
77009
|
function friendlyConfigName(parsedName, configPath) {
|
|
75252
77010
|
if (parsedName && parsedName.trim().length > 0) return parsedName.trim();
|
|
75253
|
-
return (0,
|
|
77011
|
+
return (0, import_node_path45.basename)(configPath).replace(/\.(ya?ml)$/, "");
|
|
75254
77012
|
}
|
|
75255
77013
|
function listConfigs(activeConfigPath) {
|
|
75256
|
-
const dir = (0,
|
|
77014
|
+
const dir = (0, import_node_path45.dirname)(activeConfigPath);
|
|
75257
77015
|
const entries = [];
|
|
75258
|
-
for (const fname of (0,
|
|
77016
|
+
for (const fname of (0, import_node_fs43.readdirSync)(dir)) {
|
|
75259
77017
|
if (!fname.endsWith(".yml") && !fname.endsWith(".yaml")) continue;
|
|
75260
|
-
const full = (0,
|
|
77018
|
+
const full = (0, import_node_path45.join)(dir, fname);
|
|
75261
77019
|
try {
|
|
75262
77020
|
const parsed = parseConfigFile(full);
|
|
75263
77021
|
entries.push({
|
|
@@ -75268,7 +77026,7 @@ function listConfigs(activeConfigPath) {
|
|
|
75268
77026
|
// `label` is the friendly DB name — what the user sees in the
|
|
75269
77027
|
// dropdown + settings. Falls back to the basename when unset.
|
|
75270
77028
|
label: friendlyConfigName(parsed.name, full),
|
|
75271
|
-
dbFile: (0,
|
|
77029
|
+
dbFile: (0, import_node_path45.basename)(parsed.dbPath),
|
|
75272
77030
|
active: full === activeConfigPath,
|
|
75273
77031
|
// `${LATTICE_DB:...}` and postgres:// configs resolve to a
|
|
75274
77032
|
// postgres URL; everything else is a local SQLite file. This
|
|
@@ -75282,40 +77040,40 @@ function listConfigs(activeConfigPath) {
|
|
|
75282
77040
|
return entries.sort((a6, b6) => a6.label.localeCompare(b6.label));
|
|
75283
77041
|
}
|
|
75284
77042
|
function createBlankConfig(activeConfigPath, dbName) {
|
|
75285
|
-
const dir = (0,
|
|
77043
|
+
const dir = (0, import_node_path45.dirname)(activeConfigPath);
|
|
75286
77044
|
const slug = dbName.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
75287
77045
|
if (!slug) throw new Error("Workspace name must contain at least one alphanumeric character");
|
|
75288
|
-
const configPath = (0,
|
|
75289
|
-
if ((0,
|
|
77046
|
+
const configPath = (0, import_node_path45.join)(dir, `${slug}.config.yml`);
|
|
77047
|
+
if ((0, import_node_fs43.existsSync)(configPath)) throw new Error(`Config already exists: ${slug}.config.yml`);
|
|
75290
77048
|
const yaml = `db: ./data/${slug}.db
|
|
75291
77049
|
|
|
75292
77050
|
entities: {}
|
|
75293
77051
|
`;
|
|
75294
|
-
(0,
|
|
75295
|
-
(0,
|
|
77052
|
+
(0, import_node_fs43.writeFileSync)(configPath, yaml, "utf8");
|
|
77053
|
+
(0, import_node_fs43.mkdirSync)((0, import_node_path45.join)(dir, "data"), { recursive: true });
|
|
75296
77054
|
return configPath;
|
|
75297
77055
|
}
|
|
75298
77056
|
function sqliteFileForConfig(configPath) {
|
|
75299
|
-
const dbVal = (0, import_yaml10.parseDocument)((0,
|
|
77057
|
+
const dbVal = (0, import_yaml10.parseDocument)((0, import_node_fs43.readFileSync)(configPath, "utf8")).get("db");
|
|
75300
77058
|
const raw = (typeof dbVal === "string" ? dbVal : "").trim();
|
|
75301
77059
|
if (!raw) return null;
|
|
75302
77060
|
if (isPostgresUrl(raw) || raw.startsWith("${LATTICE_DB:")) return null;
|
|
75303
77061
|
if (raw === ":memory:" || raw.startsWith("file:")) return null;
|
|
75304
|
-
return (0,
|
|
77062
|
+
return (0, import_node_path45.resolve)((0, import_node_path45.dirname)(configPath), raw);
|
|
75305
77063
|
}
|
|
75306
77064
|
function deleteDatabaseFiles(targetConfigPath) {
|
|
75307
77065
|
const sqliteFile = sqliteFileForConfig(targetConfigPath);
|
|
75308
|
-
(0,
|
|
77066
|
+
(0, import_node_fs43.unlinkSync)(targetConfigPath);
|
|
75309
77067
|
let deletedDbFile = null;
|
|
75310
|
-
if (sqliteFile && (0,
|
|
75311
|
-
(0,
|
|
77068
|
+
if (sqliteFile && (0, import_node_fs43.existsSync)(sqliteFile)) {
|
|
77069
|
+
(0, import_node_fs43.unlinkSync)(sqliteFile);
|
|
75312
77070
|
deletedDbFile = sqliteFile;
|
|
75313
77071
|
for (const suffix of ["-wal", "-shm", "-journal"]) {
|
|
75314
77072
|
const sidecar = sqliteFile + suffix;
|
|
75315
|
-
if ((0,
|
|
77073
|
+
if ((0, import_node_fs43.existsSync)(sidecar)) (0, import_node_fs43.unlinkSync)(sidecar);
|
|
75316
77074
|
}
|
|
75317
77075
|
}
|
|
75318
|
-
return { deletedConfig: (0,
|
|
77076
|
+
return { deletedConfig: (0, import_node_path45.basename)(targetConfigPath), deletedDbFile };
|
|
75319
77077
|
}
|
|
75320
77078
|
|
|
75321
77079
|
// src/gui/databases-routes.ts
|
|
@@ -75331,7 +77089,7 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
75331
77089
|
sendJson(res, {
|
|
75332
77090
|
current: {
|
|
75333
77091
|
path: active.configPath,
|
|
75334
|
-
dbFile: (0,
|
|
77092
|
+
dbFile: (0, import_node_path46.basename)(parsedActive.dbPath),
|
|
75335
77093
|
label: friendlyLabel,
|
|
75336
77094
|
kind
|
|
75337
77095
|
},
|
|
@@ -75345,8 +77103,8 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
75345
77103
|
sendJson(res, { error: "path must be a string" }, 400);
|
|
75346
77104
|
return true;
|
|
75347
77105
|
}
|
|
75348
|
-
const newPath = (0,
|
|
75349
|
-
if (!(0,
|
|
77106
|
+
const newPath = (0, import_node_path46.resolve)(body.path);
|
|
77107
|
+
if (!(0, import_node_fs44.existsSync)(newPath)) {
|
|
75350
77108
|
sendJson(res, { error: `Config not found: ${newPath}` }, 400);
|
|
75351
77109
|
return true;
|
|
75352
77110
|
}
|
|
@@ -75388,16 +77146,16 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
75388
77146
|
sendJson(res, { error: "path must be a non-empty string" }, 400);
|
|
75389
77147
|
return true;
|
|
75390
77148
|
}
|
|
75391
|
-
const target = (0,
|
|
77149
|
+
const target = (0, import_node_path46.resolve)(body.path);
|
|
75392
77150
|
const known = listConfigs(active.configPath);
|
|
75393
|
-
const match = known.find((c6) => (0,
|
|
77151
|
+
const match = known.find((c6) => (0, import_node_path46.resolve)(c6.path) === target);
|
|
75394
77152
|
if (!match) {
|
|
75395
77153
|
sendJson(res, { error: `Not a known database config: ${target}` }, 400);
|
|
75396
77154
|
return true;
|
|
75397
77155
|
}
|
|
75398
77156
|
let switchedTo = null;
|
|
75399
|
-
if ((0,
|
|
75400
|
-
const fallback = known.find((c6) => (0,
|
|
77157
|
+
if ((0, import_node_path46.resolve)(active.configPath) === target) {
|
|
77158
|
+
const fallback = known.find((c6) => (0, import_node_path46.resolve)(c6.path) !== target);
|
|
75401
77159
|
if (!fallback) {
|
|
75402
77160
|
sendJson(
|
|
75403
77161
|
res,
|
|
@@ -75488,20 +77246,26 @@ async function listenWithPortFallback(server, startPort, host) {
|
|
|
75488
77246
|
throw new Error(`No available port found starting at ${String(startPort)}`);
|
|
75489
77247
|
}
|
|
75490
77248
|
async function startGuiServer(options) {
|
|
75491
|
-
const bootConfigPath = options.configPath ? (0,
|
|
75492
|
-
const bootOutputDir = options.outputDir ? (0,
|
|
77249
|
+
const bootConfigPath = options.configPath ? (0, import_node_path47.resolve)(options.configPath) : null;
|
|
77250
|
+
const bootOutputDir = options.outputDir ? (0, import_node_path47.resolve)(options.outputDir) : null;
|
|
75493
77251
|
const startPort = options.port ?? 4317;
|
|
75494
77252
|
const host = options.host ?? "127.0.0.1";
|
|
77253
|
+
const isLoopbackHost2 = host === "localhost" || host === "::1" || host.startsWith("127.");
|
|
77254
|
+
if (!isLoopbackHost2) {
|
|
77255
|
+
console.warn(
|
|
77256
|
+
`[lattice] GUI is binding to a non-loopback address (${host}); its data routes are UNAUTHENTICATED and will be reachable from the network.`
|
|
77257
|
+
);
|
|
77258
|
+
}
|
|
75495
77259
|
const autoRender = options.autoRender ?? false;
|
|
75496
77260
|
const guiVersion = options.version ?? "";
|
|
75497
77261
|
const sessionId = crypto.randomUUID();
|
|
75498
77262
|
let updateService = null;
|
|
75499
77263
|
let activeRef = bootConfigPath && bootOutputDir ? await openConfig(bootConfigPath, bootOutputDir, autoRender, options.realtimeWatchdogMs) : null;
|
|
75500
|
-
const latticeRoot = (bootConfigPath ? findLatticeRoot((0,
|
|
77264
|
+
const latticeRoot = (bootConfigPath ? findLatticeRoot((0, import_node_path47.dirname)(bootConfigPath)) : null) ?? (options.latticeRoot ? (0, import_node_path47.resolve)(options.latticeRoot) : null);
|
|
75501
77265
|
let currentWorkspaceId = null;
|
|
75502
77266
|
if (latticeRoot && bootConfigPath) {
|
|
75503
77267
|
const launched = listWorkspaces(latticeRoot).find(
|
|
75504
|
-
(w2) => (0,
|
|
77268
|
+
(w2) => (0, import_node_path47.resolve)(resolveWorkspacePaths(latticeRoot, w2).configPath) === (0, import_node_path47.resolve)(bootConfigPath)
|
|
75505
77269
|
);
|
|
75506
77270
|
if (launched) {
|
|
75507
77271
|
currentWorkspaceId = launched.id;
|
|
@@ -75837,7 +77601,7 @@ async function startGuiServer(options) {
|
|
|
75837
77601
|
createJunction: (otherTable) => createFileJunction(active, otherTable, sessionId),
|
|
75838
77602
|
createEntity: (entity, columns) => createUserEntity(active, entity, columns, sessionId),
|
|
75839
77603
|
aggressiveness: getAggressiveness(),
|
|
75840
|
-
latticeRoot: (0,
|
|
77604
|
+
latticeRoot: (0, import_node_path47.dirname)(active.configPath),
|
|
75841
77605
|
configPath: active.configPath,
|
|
75842
77606
|
outputDir: active.outputDir,
|
|
75843
77607
|
sessionId,
|
|
@@ -75846,13 +77610,29 @@ async function startGuiServer(options) {
|
|
|
75846
77610
|
});
|
|
75847
77611
|
}
|
|
75848
77612
|
},
|
|
77613
|
+
// ── Structured-source import (apply) ──
|
|
77614
|
+
// The importer is reachable only via dropping a file in the assistant
|
|
77615
|
+
// chat; this materializes the user-confirmed proposal, re-reading the
|
|
77616
|
+
// file's bytes from its `fileId` (its retained blob).
|
|
77617
|
+
{
|
|
77618
|
+
handle: async (req2, res2) => {
|
|
77619
|
+
if (!pathname.startsWith("/api/import/")) return false;
|
|
77620
|
+
return await dispatchImportRoute(req2, res2, {
|
|
77621
|
+
db: active.db,
|
|
77622
|
+
configPath: active.configPath,
|
|
77623
|
+
latticeRoot: (0, import_node_path47.dirname)(active.configPath),
|
|
77624
|
+
validTables: active.validTables,
|
|
77625
|
+
softDeletable: active.softDeletable
|
|
77626
|
+
});
|
|
77627
|
+
}
|
|
77628
|
+
},
|
|
75849
77629
|
// ── Files: blob serving + open-in-finder ──
|
|
75850
77630
|
{
|
|
75851
77631
|
handle: async (req2, res2) => {
|
|
75852
77632
|
if (!pathname.startsWith("/api/files/")) return false;
|
|
75853
77633
|
return await dispatchFilesRoute(req2, res2, {
|
|
75854
77634
|
db: active.db,
|
|
75855
|
-
latticeRoot: (0,
|
|
77635
|
+
latticeRoot: (0, import_node_path47.dirname)(active.configPath),
|
|
75856
77636
|
configPath: active.configPath,
|
|
75857
77637
|
pathname,
|
|
75858
77638
|
method
|
|
@@ -76036,6 +77816,7 @@ ${e6.stack ?? ""}`
|
|
|
76036
77816
|
server,
|
|
76037
77817
|
port,
|
|
76038
77818
|
url,
|
|
77819
|
+
whenConverged: () => activeRef?.converged ?? Promise.resolve(),
|
|
76039
77820
|
close: () => new Promise((resolveClose, reject) => {
|
|
76040
77821
|
updateService?.stop();
|
|
76041
77822
|
for (const client of wss.clients) {
|
|
@@ -76062,9 +77843,9 @@ ${e6.stack ?? ""}`
|
|
|
76062
77843
|
}
|
|
76063
77844
|
|
|
76064
77845
|
// src/cloud/file-source-key-store.ts
|
|
76065
|
-
var
|
|
76066
|
-
var
|
|
76067
|
-
var
|
|
77846
|
+
var import_node_fs45 = require("fs");
|
|
77847
|
+
var import_node_path48 = require("path");
|
|
77848
|
+
var import_node_crypto24 = require("crypto");
|
|
76068
77849
|
var ENC_HEADER = "LATTICE-KMS-v1\n";
|
|
76069
77850
|
var SCRYPT_N = 1 << 15;
|
|
76070
77851
|
var SCRYPT_R = 8;
|
|
@@ -76081,7 +77862,7 @@ var FileSourceKeyStore = class {
|
|
|
76081
77862
|
if (!opts.path || typeof opts.path !== "string") {
|
|
76082
77863
|
throw new Error("lattice: FileSourceKeyStore requires a non-empty `path`");
|
|
76083
77864
|
}
|
|
76084
|
-
this.path = (0,
|
|
77865
|
+
this.path = (0, import_node_path48.resolve)(opts.path);
|
|
76085
77866
|
this.passphrase = opts.passphrase;
|
|
76086
77867
|
this.cache = this.load();
|
|
76087
77868
|
}
|
|
@@ -76091,7 +77872,7 @@ var FileSourceKeyStore = class {
|
|
|
76091
77872
|
getOrCreate(sourceId) {
|
|
76092
77873
|
let key = this.cache.get(sourceId);
|
|
76093
77874
|
if (!key) {
|
|
76094
|
-
key = (0,
|
|
77875
|
+
key = (0, import_node_crypto24.randomBytes)(KEY_LEN2);
|
|
76095
77876
|
this.cache.set(sourceId, key);
|
|
76096
77877
|
this.persist();
|
|
76097
77878
|
}
|
|
@@ -76114,12 +77895,12 @@ var FileSourceKeyStore = class {
|
|
|
76114
77895
|
// ── internals ────────────────────────────────────────────────────────
|
|
76115
77896
|
load() {
|
|
76116
77897
|
const out = /* @__PURE__ */ new Map();
|
|
76117
|
-
if (!(0,
|
|
76118
|
-
const dir = (0,
|
|
76119
|
-
if (!(0,
|
|
77898
|
+
if (!(0, import_node_fs45.existsSync)(this.path)) {
|
|
77899
|
+
const dir = (0, import_node_path48.dirname)(this.path);
|
|
77900
|
+
if (!(0, import_node_fs45.existsSync)(dir)) (0, import_node_fs45.mkdirSync)(dir, { recursive: true, mode: 448 });
|
|
76120
77901
|
return out;
|
|
76121
77902
|
}
|
|
76122
|
-
const raw = (0,
|
|
77903
|
+
const raw = (0, import_node_fs45.readFileSync)(this.path);
|
|
76123
77904
|
const json = this.decodeFile(raw);
|
|
76124
77905
|
for (const [sourceId, b64] of Object.entries(json)) {
|
|
76125
77906
|
try {
|
|
@@ -76135,13 +77916,13 @@ var FileSourceKeyStore = class {
|
|
|
76135
77916
|
const obj2 = {};
|
|
76136
77917
|
for (const [k6, v2] of this.cache) obj2[k6] = v2.toString("base64");
|
|
76137
77918
|
const encoded = this.encodeFile(obj2);
|
|
76138
|
-
const tmpPath = `${this.path}.tmp-${process.pid.toString()}-${(0,
|
|
76139
|
-
(0,
|
|
77919
|
+
const tmpPath = `${this.path}.tmp-${process.pid.toString()}-${(0, import_node_crypto24.randomBytes)(4).toString("hex")}`;
|
|
77920
|
+
(0, import_node_fs45.writeFileSync)(tmpPath, encoded, { mode: 384 });
|
|
76140
77921
|
try {
|
|
76141
|
-
(0,
|
|
77922
|
+
(0, import_node_fs45.chmodSync)(tmpPath, 384);
|
|
76142
77923
|
} catch {
|
|
76143
77924
|
}
|
|
76144
|
-
(0,
|
|
77925
|
+
(0, import_node_fs45.renameSync)(tmpPath, this.path);
|
|
76145
77926
|
}
|
|
76146
77927
|
decodeFile(raw) {
|
|
76147
77928
|
const text = raw.toString("utf8");
|
|
@@ -76178,14 +77959,14 @@ var FileSourceKeyStore = class {
|
|
|
76178
77959
|
if (passphrase === void 0) {
|
|
76179
77960
|
throw new Error("lattice: key file is encrypted but no passphrase was configured");
|
|
76180
77961
|
}
|
|
76181
|
-
const derived = (0,
|
|
77962
|
+
const derived = (0, import_node_crypto24.scryptSync)(passphrase, salt, KEY_LEN2, {
|
|
76182
77963
|
N: SCRYPT_N,
|
|
76183
77964
|
r: SCRYPT_R,
|
|
76184
77965
|
p: SCRYPT_P,
|
|
76185
77966
|
maxmem: 64 * 1024 * 1024
|
|
76186
77967
|
// raise Node's default 32MB cap so N=2^15 fits
|
|
76187
77968
|
});
|
|
76188
|
-
const decipher = (0,
|
|
77969
|
+
const decipher = (0, import_node_crypto24.createDecipheriv)("aes-256-gcm", derived, iv);
|
|
76189
77970
|
decipher.setAuthTag(tag);
|
|
76190
77971
|
let plaintext;
|
|
76191
77972
|
try {
|
|
@@ -76200,15 +77981,15 @@ var FileSourceKeyStore = class {
|
|
|
76200
77981
|
if (!this.passphrase) {
|
|
76201
77982
|
return Buffer.from(json, "utf8");
|
|
76202
77983
|
}
|
|
76203
|
-
const salt = (0,
|
|
76204
|
-
const iv = (0,
|
|
76205
|
-
const derived = (0,
|
|
77984
|
+
const salt = (0, import_node_crypto24.randomBytes)(SALT_LEN2);
|
|
77985
|
+
const iv = (0, import_node_crypto24.randomBytes)(IV_LEN2);
|
|
77986
|
+
const derived = (0, import_node_crypto24.scryptSync)(this.passphrase, salt, KEY_LEN2, {
|
|
76206
77987
|
N: SCRYPT_N,
|
|
76207
77988
|
r: SCRYPT_R,
|
|
76208
77989
|
p: SCRYPT_P,
|
|
76209
77990
|
maxmem: 64 * 1024 * 1024
|
|
76210
77991
|
});
|
|
76211
|
-
const cipher = (0,
|
|
77992
|
+
const cipher = (0, import_node_crypto24.createCipheriv)("aes-256-gcm", derived, iv);
|
|
76212
77993
|
const ct = Buffer.concat([cipher.update(json, "utf8"), cipher.final()]);
|
|
76213
77994
|
const tag = cipher.getAuthTag();
|
|
76214
77995
|
const body = `${salt.toString("hex")}:${iv.toString("hex")}:${Buffer.concat([ct, tag]).toString("hex")}`;
|
|
@@ -76229,6 +78010,7 @@ var FileSourceKeyStore = class {
|
|
|
76229
78010
|
DEFAULT_TYPE_ALIASES,
|
|
76230
78011
|
EMBEDDINGS_TABLE,
|
|
76231
78012
|
EmbeddingDimensionMismatchError,
|
|
78013
|
+
EmbeddingScanTooLargeError,
|
|
76232
78014
|
FileSourceKeyStore,
|
|
76233
78015
|
FoldCache,
|
|
76234
78016
|
InMemorySourceKeyStore,
|
|
@@ -76292,6 +78074,7 @@ var FileSourceKeyStore = class {
|
|
|
76292
78074
|
createS3Store,
|
|
76293
78075
|
createSQLiteStateStore,
|
|
76294
78076
|
decrypt,
|
|
78077
|
+
dedupeAndDetectViews,
|
|
76295
78078
|
defaultWorkspaceYaml,
|
|
76296
78079
|
deleteDbCredential,
|
|
76297
78080
|
deleteToken,
|
|
@@ -76299,6 +78082,9 @@ var FileSourceKeyStore = class {
|
|
|
76299
78082
|
deriveKey,
|
|
76300
78083
|
describeImage,
|
|
76301
78084
|
describePdf,
|
|
78085
|
+
detectAsOf,
|
|
78086
|
+
detectAsOfCandidates,
|
|
78087
|
+
detectAsOfColumns,
|
|
76302
78088
|
detectRetrievalRegressions,
|
|
76303
78089
|
diagnoseRetrieval,
|
|
76304
78090
|
discoverCloudTables,
|
|
@@ -76316,6 +78102,7 @@ var FileSourceKeyStore = class {
|
|
|
76316
78102
|
entityFileNames,
|
|
76317
78103
|
estimateTokens,
|
|
76318
78104
|
evaluateRetrieval,
|
|
78105
|
+
excelToRecords,
|
|
76319
78106
|
extractEdgesFromColumn,
|
|
76320
78107
|
extractObjects,
|
|
76321
78108
|
filePresignSql,
|
|
@@ -76344,6 +78131,8 @@ var FileSourceKeyStore = class {
|
|
|
76344
78131
|
hashFile,
|
|
76345
78132
|
hybridSearch,
|
|
76346
78133
|
importLegacyUserConfig,
|
|
78134
|
+
inferFieldType,
|
|
78135
|
+
inferSchema,
|
|
76347
78136
|
installCloudRls,
|
|
76348
78137
|
installCloudSettings,
|
|
76349
78138
|
installFilePresigner,
|
|
@@ -76362,15 +78151,19 @@ var FileSourceKeyStore = class {
|
|
|
76362
78151
|
loadColumnPolicy,
|
|
76363
78152
|
manifestPath,
|
|
76364
78153
|
markdownTable,
|
|
78154
|
+
matchSchemaToExisting,
|
|
78155
|
+
materializeImport,
|
|
76365
78156
|
memberGroupFor,
|
|
76366
78157
|
memberRoleName,
|
|
76367
78158
|
migrateLatticeData,
|
|
76368
78159
|
neighbors,
|
|
78160
|
+
normalizeName,
|
|
76369
78161
|
observationVisible,
|
|
76370
78162
|
observationsFromChange,
|
|
76371
78163
|
openTargetLatticeForMigration,
|
|
76372
78164
|
openUnderSource,
|
|
76373
78165
|
organizeSource,
|
|
78166
|
+
parseCellDate,
|
|
76374
78167
|
parseConfigFile,
|
|
76375
78168
|
parseConfigString,
|
|
76376
78169
|
parseMarkdownEntries,
|
|
@@ -76398,6 +78191,7 @@ var FileSourceKeyStore = class {
|
|
|
76398
78191
|
registryPath,
|
|
76399
78192
|
removeEdge,
|
|
76400
78193
|
removeEmbedding,
|
|
78194
|
+
renameEntities,
|
|
76401
78195
|
resolveActiveS3Config,
|
|
76402
78196
|
resolveLatticeRoot,
|
|
76403
78197
|
resolveProvenanceFields,
|
|
@@ -76428,6 +78222,7 @@ var FileSourceKeyStore = class {
|
|
|
76428
78222
|
setTableNeverShare,
|
|
76429
78223
|
shredSource,
|
|
76430
78224
|
slugify,
|
|
78225
|
+
sourceRecords,
|
|
76431
78226
|
startGuiServer,
|
|
76432
78227
|
storeEmbedding,
|
|
76433
78228
|
summarizeText,
|