latticesql 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/cli.js +1899 -135
- package/dist/index.cjs +1948 -153
- package/dist/index.d.cts +360 -1
- package/dist/index.d.ts +360 -1
- package/dist/index.js +1908 -127
- package/docs/api-reference.md +60 -4
- package/docs/architecture.md +24 -0
- package/docs/assistant.md +23 -0
- package/docs/examples/dashboard.html +284 -0
- package/docs/importing.md +118 -0
- package/docs/retrieval.md +31 -0
- package/package.json +7 -3
package/dist/cli.js
CHANGED
|
@@ -299,7 +299,9 @@ function withCredentialLock(fn) {
|
|
|
299
299
|
fd = openSync(lockPath, "wx");
|
|
300
300
|
break;
|
|
301
301
|
} catch (err) {
|
|
302
|
-
|
|
302
|
+
const code = err.code;
|
|
303
|
+
const contended = code === "EEXIST" || process.platform === "win32" && (code === "EPERM" || code === "EACCES");
|
|
304
|
+
if (!contended) throw err;
|
|
303
305
|
try {
|
|
304
306
|
if (Date.now() - statSync(lockPath).mtimeMs > LOCK_STALE_MS) {
|
|
305
307
|
unlinkSync(lockPath);
|
|
@@ -6277,6 +6279,7 @@ function deriveCanonicalContexts(tables) {
|
|
|
6277
6279
|
childrenOf.set(rel.table, list);
|
|
6278
6280
|
}
|
|
6279
6281
|
}
|
|
6282
|
+
const byName = new Map(tables.map((t8) => [t8.name, t8.definition]));
|
|
6280
6283
|
const out = [];
|
|
6281
6284
|
for (const { name, definition } of tables) {
|
|
6282
6285
|
const files = {};
|
|
@@ -6292,11 +6295,32 @@ function deriveCanonicalContexts(tables) {
|
|
|
6292
6295
|
};
|
|
6293
6296
|
}
|
|
6294
6297
|
for (const child of childrenOf.get(name) ?? []) {
|
|
6295
|
-
|
|
6296
|
-
|
|
6297
|
-
|
|
6298
|
-
|
|
6299
|
-
|
|
6298
|
+
const childDef = byName.get(child.table);
|
|
6299
|
+
const childBt = childDef ? belongsToRelations(childDef) : [];
|
|
6300
|
+
const [rel0, rel1] = childBt;
|
|
6301
|
+
if (childDef && rel0 && rel1 && isRenderJunction(childDef, childBt)) {
|
|
6302
|
+
const localRel = rel0.foreignKey === child.foreignKey ? rel0 : rel1;
|
|
6303
|
+
const remoteRel = localRel === rel0 ? rel1 : rel0;
|
|
6304
|
+
const fileKey = remoteRel.table === name ? `${child.table.toUpperCase()}__${remoteRel.foreignKey.toUpperCase()}.md` : `${remoteRel.table.toUpperCase()}.md`;
|
|
6305
|
+
files[fileKey] = {
|
|
6306
|
+
source: {
|
|
6307
|
+
type: "manyToMany",
|
|
6308
|
+
junctionTable: child.table,
|
|
6309
|
+
localKey: localRel.foreignKey,
|
|
6310
|
+
remoteKey: remoteRel.foreignKey,
|
|
6311
|
+
remoteTable: remoteRel.table,
|
|
6312
|
+
references: remoteRel.references ?? "id"
|
|
6313
|
+
},
|
|
6314
|
+
render: renderRelated(remoteRel.table),
|
|
6315
|
+
omitIfEmpty: true
|
|
6316
|
+
};
|
|
6317
|
+
} else {
|
|
6318
|
+
files[`${child.table.toUpperCase()}.md`] = {
|
|
6319
|
+
source: { type: "hasMany", table: child.table, foreignKey: child.foreignKey },
|
|
6320
|
+
render: renderRelated(child.table),
|
|
6321
|
+
omitIfEmpty: true
|
|
6322
|
+
};
|
|
6323
|
+
}
|
|
6300
6324
|
}
|
|
6301
6325
|
out.push({
|
|
6302
6326
|
table: name,
|
|
@@ -6309,6 +6333,15 @@ function deriveCanonicalContexts(tables) {
|
|
|
6309
6333
|
}
|
|
6310
6334
|
return out;
|
|
6311
6335
|
}
|
|
6336
|
+
function isRenderJunction(def, bt) {
|
|
6337
|
+
if (bt.length !== 2) return false;
|
|
6338
|
+
const fks = new Set(bt.map((r6) => r6.foreignKey));
|
|
6339
|
+
if (fks.size !== 2) return false;
|
|
6340
|
+
const pk = Array.isArray(def.primaryKey) ? def.primaryKey : def.primaryKey != null ? [def.primaryKey] : [];
|
|
6341
|
+
if (pk.length === 2 && pk.every((c6) => fks.has(c6))) return true;
|
|
6342
|
+
const SYSTEM2 = /* @__PURE__ */ new Set(["id", "created_at", "updated_at", "deleted_at"]);
|
|
6343
|
+
return Object.keys(def.columns).every((c6) => fks.has(c6) || SYSTEM2.has(c6));
|
|
6344
|
+
}
|
|
6312
6345
|
function belongsToRelations(def) {
|
|
6313
6346
|
return Object.values(def.relations ?? {}).filter(
|
|
6314
6347
|
(r6) => r6.type === "belongsTo"
|
|
@@ -6702,6 +6735,19 @@ var init_vector_index = __esm({
|
|
|
6702
6735
|
}
|
|
6703
6736
|
});
|
|
6704
6737
|
|
|
6738
|
+
// src/search/limits.ts
|
|
6739
|
+
function clampTopK(topK) {
|
|
6740
|
+
if (!Number.isFinite(topK)) return 1;
|
|
6741
|
+
return Math.min(Math.max(1, Math.floor(topK)), SEARCH_TOPK_MAX);
|
|
6742
|
+
}
|
|
6743
|
+
var SEARCH_TOPK_MAX;
|
|
6744
|
+
var init_limits = __esm({
|
|
6745
|
+
"src/search/limits.ts"() {
|
|
6746
|
+
"use strict";
|
|
6747
|
+
SEARCH_TOPK_MAX = 1e3;
|
|
6748
|
+
}
|
|
6749
|
+
});
|
|
6750
|
+
|
|
6705
6751
|
// src/search/embeddings.ts
|
|
6706
6752
|
async function ensureEmbeddingsTable(adapter) {
|
|
6707
6753
|
let cols = [];
|
|
@@ -6848,9 +6894,10 @@ function cosineSimilarity(a6, b6) {
|
|
|
6848
6894
|
}
|
|
6849
6895
|
async function searchByEmbedding(adapter, table, queryText, config, topK, minScore, pkColumn = "id") {
|
|
6850
6896
|
const queryVector = await config.embed(queryText);
|
|
6897
|
+
const k6 = clampTopK(topK);
|
|
6851
6898
|
let ranked;
|
|
6852
6899
|
if (await vectorIndexAvailable(adapter) && await hasVectorIndex(adapter, table)) {
|
|
6853
|
-
const hits = await searchVectorIndex(adapter, table, queryVector,
|
|
6900
|
+
const hits = await searchVectorIndex(adapter, table, queryVector, k6 * 4, minScore);
|
|
6854
6901
|
ranked = hits.map((h6) => ({
|
|
6855
6902
|
pk: h6.pk,
|
|
6856
6903
|
score: h6.score,
|
|
@@ -6858,7 +6905,7 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
6858
6905
|
content: h6.content
|
|
6859
6906
|
}));
|
|
6860
6907
|
} else {
|
|
6861
|
-
ranked = await scanChunks(adapter, table, queryVector, minScore);
|
|
6908
|
+
ranked = await scanChunks(adapter, table, queryVector, minScore, config.maxScanChunks);
|
|
6862
6909
|
}
|
|
6863
6910
|
const bestByRow = /* @__PURE__ */ new Map();
|
|
6864
6911
|
for (const r6 of ranked) {
|
|
@@ -6883,11 +6930,20 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
6883
6930
|
if (r6.content !== null) result.matchedContent = r6.content;
|
|
6884
6931
|
}
|
|
6885
6932
|
results.push(result);
|
|
6886
|
-
if (results.length >=
|
|
6933
|
+
if (results.length >= k6) break;
|
|
6887
6934
|
}
|
|
6888
6935
|
return results;
|
|
6889
6936
|
}
|
|
6890
|
-
async function scanChunks(adapter, table, queryVector, minScore) {
|
|
6937
|
+
async function scanChunks(adapter, table, queryVector, minScore, maxScanChunks) {
|
|
6938
|
+
if (maxScanChunks !== void 0) {
|
|
6939
|
+
const countRows = await allAsyncOrSync(
|
|
6940
|
+
adapter,
|
|
6941
|
+
`SELECT COUNT(*) AS n FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
6942
|
+
[table]
|
|
6943
|
+
);
|
|
6944
|
+
const n3 = Number(countRows[0]?.n ?? 0);
|
|
6945
|
+
if (n3 > maxScanChunks) throw new EmbeddingScanTooLargeError(table, n3, maxScanChunks);
|
|
6946
|
+
}
|
|
6891
6947
|
const stored = await allAsyncOrSync(
|
|
6892
6948
|
adapter,
|
|
6893
6949
|
`SELECT "row_pk", "chunk_index", "content", "embedding", "vec_dim" FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
@@ -6997,13 +7053,14 @@ async function refreshEmbeddings(adapter, table, config, pkColumn = "id", opts =
|
|
|
6997
7053
|
}
|
|
6998
7054
|
return { embedded, skipped, removed };
|
|
6999
7055
|
}
|
|
7000
|
-
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError;
|
|
7056
|
+
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError, EmbeddingScanTooLargeError;
|
|
7001
7057
|
var init_embeddings = __esm({
|
|
7002
7058
|
"src/search/embeddings.ts"() {
|
|
7003
7059
|
"use strict";
|
|
7004
7060
|
init_adapter();
|
|
7005
7061
|
init_chunking();
|
|
7006
7062
|
init_vector_index();
|
|
7063
|
+
init_limits();
|
|
7007
7064
|
EMBEDDINGS_TABLE = "_lattice_embeddings";
|
|
7008
7065
|
EmbeddingDimensionMismatchError = class extends Error {
|
|
7009
7066
|
constructor(table, expected, found) {
|
|
@@ -7016,6 +7073,17 @@ var init_embeddings = __esm({
|
|
|
7016
7073
|
this.name = "EmbeddingDimensionMismatchError";
|
|
7017
7074
|
}
|
|
7018
7075
|
};
|
|
7076
|
+
EmbeddingScanTooLargeError = class extends Error {
|
|
7077
|
+
constructor(table, found, limit) {
|
|
7078
|
+
super(
|
|
7079
|
+
`Embedding scan on "${table}" would read ${String(found)} stored chunk vectors, over the configured maxScanChunks of ${String(limit)}. Add a native vector index (pgvector) for this table or raise maxScanChunks \u2014 Lattice will not silently truncate the scan, which would return incomplete results.`
|
|
7080
|
+
);
|
|
7081
|
+
this.table = table;
|
|
7082
|
+
this.found = found;
|
|
7083
|
+
this.limit = limit;
|
|
7084
|
+
this.name = "EmbeddingScanTooLargeError";
|
|
7085
|
+
}
|
|
7086
|
+
};
|
|
7019
7087
|
}
|
|
7020
7088
|
});
|
|
7021
7089
|
|
|
@@ -7368,7 +7436,7 @@ async function fetchLiveRows2(adapter, table, ids, pkColumn) {
|
|
|
7368
7436
|
return out;
|
|
7369
7437
|
}
|
|
7370
7438
|
async function hybridSearch(adapter, table, query, opts = {}) {
|
|
7371
|
-
const topK = opts.topK ?? 10;
|
|
7439
|
+
const topK = clampTopK(opts.topK ?? 10);
|
|
7372
7440
|
const rrfK = opts.rrfK ?? 60;
|
|
7373
7441
|
const pool = opts.poolSize ?? Math.max(topK * 4, 20);
|
|
7374
7442
|
const pkColumn = opts.pkColumn ?? "id";
|
|
@@ -7469,6 +7537,7 @@ var init_hybrid = __esm({
|
|
|
7469
7537
|
init_fts();
|
|
7470
7538
|
init_ranking();
|
|
7471
7539
|
init_rerank();
|
|
7540
|
+
init_limits();
|
|
7472
7541
|
}
|
|
7473
7542
|
});
|
|
7474
7543
|
|
|
@@ -8196,6 +8265,26 @@ RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
|
8196
8265
|
);
|
|
8197
8266
|
$fn$;
|
|
8198
8267
|
|
|
8268
|
+
-- Delete-event visibility, decided from the PRE-DELETE snapshot the delete trigger
|
|
8269
|
+
-- captures (the live row + its ownership record are gone after a delete, so
|
|
8270
|
+
-- lattice_row_visible can't be used). Keyed on session_user, SECURITY DEFINER \u2014
|
|
8271
|
+
-- the same per-recipient gate. MUST MIRROR lattice_row_visible's rule: the row is
|
|
8272
|
+
-- visible iff this member owned it, OR it was 'everyone', OR it was 'custom' and
|
|
8273
|
+
-- this member was a grantee. A NULL owner snapshot (a legacy delete emitted before
|
|
8274
|
+
-- the snapshot columns, or a row with no ownership record) yields false \u2014 fail
|
|
8275
|
+
-- closed, never forward. (tests/integration assert this agrees with
|
|
8276
|
+
-- lattice_row_visible for all three visibility states \u2014 the no-drift guard.)
|
|
8277
|
+
CREATE OR REPLACE FUNCTION lattice_delete_visible(
|
|
8278
|
+
p_owner_role text, p_visibility text, p_grantees text[]
|
|
8279
|
+
)
|
|
8280
|
+
RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
8281
|
+
SELECT p_owner_role IS NOT NULL AND (
|
|
8282
|
+
p_owner_role = session_user
|
|
8283
|
+
OR p_visibility = 'everyone'
|
|
8284
|
+
OR (p_visibility = 'custom' AND session_user = ANY(COALESCE(p_grantees, ARRAY[]::text[])))
|
|
8285
|
+
);
|
|
8286
|
+
$fn$;
|
|
8287
|
+
|
|
8199
8288
|
-- Shared owner gate: raises unless the connected member owns (p_table, p_pk).
|
|
8200
8289
|
-- p_action is spliced into the message so every caller keeps its exact wording.
|
|
8201
8290
|
-- SECURITY DEFINER + session_user (never current_user), the cloud identity invariant.
|
|
@@ -8370,6 +8459,14 @@ CREATE TABLE IF NOT EXISTS "__lattice_changes" (
|
|
|
8370
8459
|
"created_at" timestamptz NOT NULL DEFAULT now()
|
|
8371
8460
|
);
|
|
8372
8461
|
|
|
8462
|
+
-- Pre-delete visibility snapshot columns (added to existing clouds via ADD COLUMN
|
|
8463
|
+
-- IF NOT EXISTS). A delete event carries the row's visibility AT DELETE TIME so the
|
|
8464
|
+
-- live fan-out can gate it per recipient even though the ownership record is gone.
|
|
8465
|
+
-- NULL on upserts.
|
|
8466
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_owner_role" text;
|
|
8467
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_visibility" text;
|
|
8468
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_grantees" text[];
|
|
8469
|
+
|
|
8373
8470
|
CREATE OR REPLACE FUNCTION lattice_notify_change() RETURNS trigger
|
|
8374
8471
|
LANGUAGE plpgsql AS $fn$
|
|
8375
8472
|
BEGIN
|
|
@@ -8379,7 +8476,10 @@ BEGIN
|
|
|
8379
8476
|
'pk', NEW."pk",
|
|
8380
8477
|
'op', NEW."op",
|
|
8381
8478
|
'owner_role', NEW."owner_role",
|
|
8382
|
-
'created_at', NEW."created_at"
|
|
8479
|
+
'created_at', NEW."created_at",
|
|
8480
|
+
'del_owner_role', NEW."del_owner_role",
|
|
8481
|
+
'del_visibility', NEW."del_visibility",
|
|
8482
|
+
'del_grantees', NEW."del_grantees"
|
|
8383
8483
|
)::text);
|
|
8384
8484
|
RETURN NEW;
|
|
8385
8485
|
END $fn$;
|
|
@@ -8585,10 +8685,22 @@ BEGIN
|
|
|
8585
8685
|
VALUES (${lit}, ${pkNew}, 'upsert', session_user);
|
|
8586
8686
|
RETURN NEW;
|
|
8587
8687
|
ELSIF TG_OP = 'DELETE' THEN
|
|
8688
|
+
-- Snapshot the row's visibility BEFORE the cascade removes its ownership +
|
|
8689
|
+
-- grant records, so the realtime fan-out can gate the delete event per
|
|
8690
|
+
-- recipient (the live predicate can't \u2014 these records are gone post-delete).
|
|
8691
|
+
-- The grantee list is captured here because the grant rows are deleted in the
|
|
8692
|
+
-- same statement below; after that the 'custom' audience is unrecoverable.
|
|
8693
|
+
INSERT INTO "__lattice_changes"
|
|
8694
|
+
("table_name","pk","op","owner_role","del_owner_role","del_visibility","del_grantees")
|
|
8695
|
+
VALUES (${lit}, ${pkOld}, 'delete', session_user,
|
|
8696
|
+
(SELECT o."owner_role" FROM "__lattice_owners" o
|
|
8697
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
8698
|
+
(SELECT o."visibility" FROM "__lattice_owners" o
|
|
8699
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
8700
|
+
COALESCE((SELECT array_agg(g."grantee_role") FROM "__lattice_row_grants" g
|
|
8701
|
+
WHERE g."table_name" = ${lit} AND g."pk" = ${pkOld}), ARRAY[]::text[]));
|
|
8588
8702
|
DELETE FROM "__lattice_owners" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8589
8703
|
DELETE FROM "__lattice_row_grants" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8590
|
-
INSERT INTO "__lattice_changes" ("table_name","pk","op","owner_role")
|
|
8591
|
-
VALUES (${lit}, ${pkOld}, 'delete', session_user);
|
|
8592
8704
|
RETURN OLD;
|
|
8593
8705
|
END IF;
|
|
8594
8706
|
RETURN NEW;
|
|
@@ -11830,7 +11942,7 @@ function parsePageParam(raw, kind) {
|
|
|
11830
11942
|
}
|
|
11831
11943
|
function readJson(req, opts = {}) {
|
|
11832
11944
|
const maxBytes = opts.maxBytes ?? DEFAULT_BODY_MAX_BYTES;
|
|
11833
|
-
return new Promise((
|
|
11945
|
+
return new Promise((resolve17, reject) => {
|
|
11834
11946
|
let raw = "";
|
|
11835
11947
|
let overflowed = false;
|
|
11836
11948
|
req.setEncoding("utf8");
|
|
@@ -11846,7 +11958,7 @@ function readJson(req, opts = {}) {
|
|
|
11846
11958
|
req.on("end", () => {
|
|
11847
11959
|
if (overflowed) return;
|
|
11848
11960
|
try {
|
|
11849
|
-
|
|
11961
|
+
resolve17(raw ? JSON.parse(raw) : {});
|
|
11850
11962
|
} catch {
|
|
11851
11963
|
reject(new Error("Invalid JSON body"));
|
|
11852
11964
|
}
|
|
@@ -11866,11 +11978,12 @@ ${err.stack ?? ""}`);
|
|
|
11866
11978
|
sendJson(res, { error: err.message }, status);
|
|
11867
11979
|
}
|
|
11868
11980
|
}
|
|
11869
|
-
var DEFAULT_BODY_MAX_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
11981
|
+
var DEFAULT_BODY_MAX_BYTES, MAX_INGEST_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
11870
11982
|
var init_http = __esm({
|
|
11871
11983
|
"src/gui/http.ts"() {
|
|
11872
11984
|
"use strict";
|
|
11873
11985
|
DEFAULT_BODY_MAX_BYTES = 1e6;
|
|
11986
|
+
MAX_INGEST_BYTES = 5e7;
|
|
11874
11987
|
MAX_ROWS_PAGE = 1e3;
|
|
11875
11988
|
DEFAULT_ROWS_PAGE = 500;
|
|
11876
11989
|
BodyTooLargeError = class extends Error {
|
|
@@ -13430,7 +13543,7 @@ var init_oauth = __esm({
|
|
|
13430
13543
|
|
|
13431
13544
|
// src/gui/assistant-routes.ts
|
|
13432
13545
|
function readBuffer(req, maxBytes = 25e6) {
|
|
13433
|
-
return new Promise((
|
|
13546
|
+
return new Promise((resolve17, reject) => {
|
|
13434
13547
|
const chunks = [];
|
|
13435
13548
|
let size = 0;
|
|
13436
13549
|
req.on("data", (c6) => {
|
|
@@ -13439,7 +13552,7 @@ function readBuffer(req, maxBytes = 25e6) {
|
|
|
13439
13552
|
else chunks.push(c6);
|
|
13440
13553
|
});
|
|
13441
13554
|
req.on("end", () => {
|
|
13442
|
-
|
|
13555
|
+
resolve17(Buffer.concat(chunks));
|
|
13443
13556
|
});
|
|
13444
13557
|
req.on("error", reject);
|
|
13445
13558
|
});
|
|
@@ -14740,7 +14853,7 @@ async function takeHostSlot(host, minIntervalMs = urlIngestConfig().hostMinInter
|
|
|
14740
14853
|
const earliest = Math.max(now2, hostNextAllowed.get(key) ?? 0);
|
|
14741
14854
|
hostNextAllowed.set(key, earliest + minIntervalMs);
|
|
14742
14855
|
const wait = earliest - now2;
|
|
14743
|
-
if (wait > 0) await new Promise((
|
|
14856
|
+
if (wait > 0) await new Promise((resolve17) => setTimeout(resolve17, wait));
|
|
14744
14857
|
}
|
|
14745
14858
|
var Semaphore, FetchBudget, sharedGate, hostNextAllowed;
|
|
14746
14859
|
var init_fetch_policy = __esm({
|
|
@@ -14756,7 +14869,7 @@ var init_fetch_policy = __esm({
|
|
|
14756
14869
|
if (this.permits > 0) {
|
|
14757
14870
|
this.permits -= 1;
|
|
14758
14871
|
} else {
|
|
14759
|
-
await new Promise((
|
|
14872
|
+
await new Promise((resolve17) => this.waiters.push(resolve17));
|
|
14760
14873
|
}
|
|
14761
14874
|
let released = false;
|
|
14762
14875
|
return () => {
|
|
@@ -14993,8 +15106,8 @@ function fileContentGroups(rows, fuzzy, threshold) {
|
|
|
14993
15106
|
const t8 = get(r6, "extracted_text");
|
|
14994
15107
|
return typeof t8 === "string" && t8.trim().length > 0;
|
|
14995
15108
|
}).map((r6) => {
|
|
14996
|
-
const
|
|
14997
|
-
const key = fuzzy ? "txt:" +
|
|
15109
|
+
const norm3 = normalizeText(get(r6, "extracted_text"));
|
|
15110
|
+
const key = fuzzy ? "txt:" + norm3.slice(0, 2e3) : "txt:" + createHash5("sha256").update(norm3).digest("hex");
|
|
14998
15111
|
return { id: String(get(r6, "id")), key, createdAt: cellStrOrNull(get(r6, "created_at")) };
|
|
14999
15112
|
});
|
|
15000
15113
|
const txtGroups = findDuplicateGroups(txtItems, {
|
|
@@ -18972,7 +19085,7 @@ var init_sleep = __esm({
|
|
|
18972
19085
|
"node_modules/@smithy/core/dist-es/submodules/client/util-waiter/utils/sleep.js"() {
|
|
18973
19086
|
"use strict";
|
|
18974
19087
|
sleep = (seconds) => {
|
|
18975
|
-
return new Promise((
|
|
19088
|
+
return new Promise((resolve17) => setTimeout(resolve17, seconds * 1e3));
|
|
18976
19089
|
};
|
|
18977
19090
|
}
|
|
18978
19091
|
});
|
|
@@ -19141,8 +19254,8 @@ var init_createWaiter = __esm({
|
|
|
19141
19254
|
init_waiter2();
|
|
19142
19255
|
abortTimeout = (abortSignal) => {
|
|
19143
19256
|
let onAbort;
|
|
19144
|
-
const promise = new Promise((
|
|
19145
|
-
onAbort = () =>
|
|
19257
|
+
const promise = new Promise((resolve17) => {
|
|
19258
|
+
onAbort = () => resolve17({ state: WaiterState.ABORTED });
|
|
19146
19259
|
if (typeof abortSignal.addEventListener === "function") {
|
|
19147
19260
|
abortSignal.addEventListener("abort", onAbort);
|
|
19148
19261
|
} else {
|
|
@@ -22024,7 +22137,7 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
22024
22137
|
};
|
|
22025
22138
|
imdsHttpGet = async ({ hostname, path: path2 }) => {
|
|
22026
22139
|
const { request } = await import("http");
|
|
22027
|
-
return new Promise((
|
|
22140
|
+
return new Promise((resolve17, reject) => {
|
|
22028
22141
|
const req = request({
|
|
22029
22142
|
method: "GET",
|
|
22030
22143
|
hostname: hostname.replace(/^\[(.+)]$/, "$1"),
|
|
@@ -22050,7 +22163,7 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
22050
22163
|
const chunks = [];
|
|
22051
22164
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
22052
22165
|
res.on("end", () => {
|
|
22053
|
-
|
|
22166
|
+
resolve17(Buffer.concat(chunks));
|
|
22054
22167
|
req.destroy();
|
|
22055
22168
|
});
|
|
22056
22169
|
});
|
|
@@ -23810,7 +23923,7 @@ async function collectStream(stream) {
|
|
|
23810
23923
|
return collected;
|
|
23811
23924
|
}
|
|
23812
23925
|
function readToBase64(blob) {
|
|
23813
|
-
return new Promise((
|
|
23926
|
+
return new Promise((resolve17, reject) => {
|
|
23814
23927
|
const reader = new FileReader();
|
|
23815
23928
|
reader.onloadend = () => {
|
|
23816
23929
|
if (reader.readyState !== 2) {
|
|
@@ -23819,7 +23932,7 @@ function readToBase64(blob) {
|
|
|
23819
23932
|
const result = reader.result ?? "";
|
|
23820
23933
|
const commaIndex = result.indexOf(",");
|
|
23821
23934
|
const dataOffset = commaIndex > -1 ? commaIndex + 1 : result.length;
|
|
23822
|
-
|
|
23935
|
+
resolve17(result.substring(dataOffset));
|
|
23823
23936
|
};
|
|
23824
23937
|
reader.onabort = () => reject(new Error("Read aborted"));
|
|
23825
23938
|
reader.onerror = () => reject(reader.error);
|
|
@@ -23947,7 +24060,7 @@ var init_stream_collector = __esm({
|
|
|
23947
24060
|
if (isReadableStreamInstance(stream)) {
|
|
23948
24061
|
return collectReadableStream(stream);
|
|
23949
24062
|
}
|
|
23950
|
-
return new Promise((
|
|
24063
|
+
return new Promise((resolve17, reject) => {
|
|
23951
24064
|
const collector = new Collector();
|
|
23952
24065
|
stream.pipe(collector);
|
|
23953
24066
|
stream.on("error", (err) => {
|
|
@@ -23957,7 +24070,7 @@ var init_stream_collector = __esm({
|
|
|
23957
24070
|
collector.on("error", reject);
|
|
23958
24071
|
collector.on("finish", function() {
|
|
23959
24072
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
23960
|
-
|
|
24073
|
+
resolve17(bytes);
|
|
23961
24074
|
});
|
|
23962
24075
|
});
|
|
23963
24076
|
};
|
|
@@ -24104,11 +24217,11 @@ var init_SerdeContext = __esm({
|
|
|
24104
24217
|
// node_modules/tslib/tslib.es6.mjs
|
|
24105
24218
|
function __awaiter(thisArg, _arguments, P2, generator) {
|
|
24106
24219
|
function adopt(value) {
|
|
24107
|
-
return value instanceof P2 ? value : new P2(function(
|
|
24108
|
-
|
|
24220
|
+
return value instanceof P2 ? value : new P2(function(resolve17) {
|
|
24221
|
+
resolve17(value);
|
|
24109
24222
|
});
|
|
24110
24223
|
}
|
|
24111
|
-
return new (P2 || (P2 = Promise))(function(
|
|
24224
|
+
return new (P2 || (P2 = Promise))(function(resolve17, reject) {
|
|
24112
24225
|
function fulfilled(value) {
|
|
24113
24226
|
try {
|
|
24114
24227
|
step(generator.next(value));
|
|
@@ -24124,7 +24237,7 @@ function __awaiter(thisArg, _arguments, P2, generator) {
|
|
|
24124
24237
|
}
|
|
24125
24238
|
}
|
|
24126
24239
|
function step(result) {
|
|
24127
|
-
result.done ?
|
|
24240
|
+
result.done ? resolve17(result.value) : adopt(result.value).then(fulfilled, rejected);
|
|
24128
24241
|
}
|
|
24129
24242
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
24130
24243
|
});
|
|
@@ -25321,7 +25434,7 @@ async function* readableToIterable(readStream) {
|
|
|
25321
25434
|
streamEnded = true;
|
|
25322
25435
|
});
|
|
25323
25436
|
while (!generationEnded) {
|
|
25324
|
-
const value = await new Promise((
|
|
25437
|
+
const value = await new Promise((resolve17) => setTimeout(() => resolve17(records.shift()), 0));
|
|
25325
25438
|
if (value) {
|
|
25326
25439
|
yield value;
|
|
25327
25440
|
}
|
|
@@ -26866,7 +26979,7 @@ var init_retryMiddleware = __esm({
|
|
|
26866
26979
|
init_constants5();
|
|
26867
26980
|
init_parseRetryAfterHeader();
|
|
26868
26981
|
init_util2();
|
|
26869
|
-
cooldown = (ms) => new Promise((
|
|
26982
|
+
cooldown = (ms) => new Promise((resolve17) => setTimeout(resolve17, ms));
|
|
26870
26983
|
isRetryStrategyV2 = (retryStrategy) => typeof retryStrategy.acquireInitialRetryToken !== "undefined" && typeof retryStrategy.refreshRetryTokenForRetry !== "undefined" && typeof retryStrategy.recordSuccess !== "undefined";
|
|
26871
26984
|
getRetryErrorInfo = (error, logger2) => {
|
|
26872
26985
|
const errorInfo = {
|
|
@@ -26965,7 +27078,7 @@ var init_DefaultRateLimiter = __esm({
|
|
|
26965
27078
|
this.refillTokenBucket();
|
|
26966
27079
|
while (amount > this.availableTokens) {
|
|
26967
27080
|
const delay = (amount - this.availableTokens) / this.fillRate * 1e3;
|
|
26968
|
-
await new Promise((
|
|
27081
|
+
await new Promise((resolve17) => _DefaultRateLimiter.setTimeoutFn(resolve17, delay));
|
|
26969
27082
|
this.refillTokenBucket();
|
|
26970
27083
|
}
|
|
26971
27084
|
this.availableTokens = this.availableTokens - amount;
|
|
@@ -30635,8 +30748,8 @@ var init_SignatureV4 = __esm({
|
|
|
30635
30748
|
priorSignature: signableMessage.priorSignature,
|
|
30636
30749
|
eventStreamCredentials
|
|
30637
30750
|
});
|
|
30638
|
-
return promise.then((
|
|
30639
|
-
return { message: signableMessage.message, signature };
|
|
30751
|
+
return promise.then((signature2) => {
|
|
30752
|
+
return { message: signableMessage.message, signature: signature2 };
|
|
30640
30753
|
});
|
|
30641
30754
|
}
|
|
30642
30755
|
async signString(stringToSign, { signingDate = /* @__PURE__ */ new Date(), signingRegion, signingService, eventStreamCredentials } = {}) {
|
|
@@ -30664,8 +30777,8 @@ var init_SignatureV4 = __esm({
|
|
|
30664
30777
|
request.headers[SHA256_HEADER] = payloadHash;
|
|
30665
30778
|
}
|
|
30666
30779
|
const canonicalHeaders = getCanonicalHeaders(request, unsignableHeaders, signableHeaders);
|
|
30667
|
-
const
|
|
30668
|
-
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${
|
|
30780
|
+
const signature2 = await this.getSignature(longDate, scope, this.getSigningKey(credentials, region, shortDate, signingService), this.createCanonicalRequest(request, canonicalHeaders, payloadHash));
|
|
30781
|
+
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${signature2}`;
|
|
30669
30782
|
return request;
|
|
30670
30783
|
}
|
|
30671
30784
|
async getSignature(longDate, credentialScope, keyPromise, canonicalRequest) {
|
|
@@ -46253,7 +46366,7 @@ var init_node_http = __esm({
|
|
|
46253
46366
|
|
|
46254
46367
|
// node_modules/@smithy/credential-provider-imds/dist-es/remoteProvider/httpRequest.js
|
|
46255
46368
|
function httpRequest(options) {
|
|
46256
|
-
return new Promise((
|
|
46369
|
+
return new Promise((resolve17, reject) => {
|
|
46257
46370
|
const req = node_http.request({
|
|
46258
46371
|
method: "GET",
|
|
46259
46372
|
...options,
|
|
@@ -46278,7 +46391,7 @@ function httpRequest(options) {
|
|
|
46278
46391
|
chunks.push(chunk);
|
|
46279
46392
|
});
|
|
46280
46393
|
res.on("end", () => {
|
|
46281
|
-
|
|
46394
|
+
resolve17(Buffer.concat(chunks));
|
|
46282
46395
|
req.destroy();
|
|
46283
46396
|
});
|
|
46284
46397
|
});
|
|
@@ -46923,21 +47036,21 @@ async function writeRequestBody(httpRequest2, request, maxContinueTimeoutMs = MI
|
|
|
46923
47036
|
let sendBody = true;
|
|
46924
47037
|
if (!externalAgent && expect === "100-continue") {
|
|
46925
47038
|
sendBody = await Promise.race([
|
|
46926
|
-
new Promise((
|
|
46927
|
-
timeoutId = Number(timing.setTimeout(() =>
|
|
47039
|
+
new Promise((resolve17) => {
|
|
47040
|
+
timeoutId = Number(timing.setTimeout(() => resolve17(true), Math.max(MIN_WAIT_TIME, maxContinueTimeoutMs)));
|
|
46928
47041
|
}),
|
|
46929
|
-
new Promise((
|
|
47042
|
+
new Promise((resolve17) => {
|
|
46930
47043
|
httpRequest2.on("continue", () => {
|
|
46931
47044
|
timing.clearTimeout(timeoutId);
|
|
46932
|
-
|
|
47045
|
+
resolve17(true);
|
|
46933
47046
|
});
|
|
46934
47047
|
httpRequest2.on("response", () => {
|
|
46935
47048
|
timing.clearTimeout(timeoutId);
|
|
46936
|
-
|
|
47049
|
+
resolve17(false);
|
|
46937
47050
|
});
|
|
46938
47051
|
httpRequest2.on("error", () => {
|
|
46939
47052
|
timing.clearTimeout(timeoutId);
|
|
46940
|
-
|
|
47053
|
+
resolve17(false);
|
|
46941
47054
|
});
|
|
46942
47055
|
})
|
|
46943
47056
|
]);
|
|
@@ -47035,13 +47148,13 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
47035
47148
|
return socketWarningTimestamp;
|
|
47036
47149
|
}
|
|
47037
47150
|
constructor(options) {
|
|
47038
|
-
this.configProvider = new Promise((
|
|
47151
|
+
this.configProvider = new Promise((resolve17, reject) => {
|
|
47039
47152
|
if (typeof options === "function") {
|
|
47040
47153
|
options().then((_options) => {
|
|
47041
|
-
|
|
47154
|
+
resolve17(this.resolveDefaultConfig(_options));
|
|
47042
47155
|
}).catch(reject);
|
|
47043
47156
|
} else {
|
|
47044
|
-
|
|
47157
|
+
resolve17(this.resolveDefaultConfig(options));
|
|
47045
47158
|
}
|
|
47046
47159
|
});
|
|
47047
47160
|
}
|
|
@@ -47072,7 +47185,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
47072
47185
|
timing.clearTimeout(socketTimeoutId);
|
|
47073
47186
|
timing.clearTimeout(keepAliveTimeoutId);
|
|
47074
47187
|
};
|
|
47075
|
-
const
|
|
47188
|
+
const resolve17 = async (arg) => {
|
|
47076
47189
|
await writeRequestBodyPromise;
|
|
47077
47190
|
clearTimeouts();
|
|
47078
47191
|
_resolve(arg);
|
|
@@ -47136,7 +47249,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
47136
47249
|
headers: getTransformedHeaders(res.headers),
|
|
47137
47250
|
body: res
|
|
47138
47251
|
});
|
|
47139
|
-
|
|
47252
|
+
resolve17({ response: httpResponse });
|
|
47140
47253
|
});
|
|
47141
47254
|
req.on("error", (err) => {
|
|
47142
47255
|
if (NODEJS_TIMEOUT_ERROR_CODES2.includes(err.code)) {
|
|
@@ -47288,7 +47401,7 @@ var init_stream_collector2 = __esm({
|
|
|
47288
47401
|
if (isReadableStreamInstance2(stream)) {
|
|
47289
47402
|
return collectReadableStream2(stream);
|
|
47290
47403
|
}
|
|
47291
|
-
return new Promise((
|
|
47404
|
+
return new Promise((resolve17, reject) => {
|
|
47292
47405
|
const collector = new Collector2();
|
|
47293
47406
|
stream.pipe(collector);
|
|
47294
47407
|
stream.on("error", (err) => {
|
|
@@ -47298,7 +47411,7 @@ var init_stream_collector2 = __esm({
|
|
|
47298
47411
|
collector.on("error", reject);
|
|
47299
47412
|
collector.on("finish", function() {
|
|
47300
47413
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
47301
|
-
|
|
47414
|
+
resolve17(bytes);
|
|
47302
47415
|
});
|
|
47303
47416
|
});
|
|
47304
47417
|
};
|
|
@@ -47420,7 +47533,7 @@ var init_retry_wrapper = __esm({
|
|
|
47420
47533
|
try {
|
|
47421
47534
|
return await toRetry();
|
|
47422
47535
|
} catch (e6) {
|
|
47423
|
-
await new Promise((
|
|
47536
|
+
await new Promise((resolve17) => setTimeout(resolve17, delayMs));
|
|
47424
47537
|
}
|
|
47425
47538
|
}
|
|
47426
47539
|
return await toRetry();
|
|
@@ -52655,14 +52768,14 @@ var init_readableStreamHasher = __esm({
|
|
|
52655
52768
|
const hash = new hashCtor();
|
|
52656
52769
|
const hashCalculator = new HashCalculator(hash);
|
|
52657
52770
|
readableStream.pipe(hashCalculator);
|
|
52658
|
-
return new Promise((
|
|
52771
|
+
return new Promise((resolve17, reject) => {
|
|
52659
52772
|
readableStream.on("error", (err) => {
|
|
52660
52773
|
hashCalculator.end();
|
|
52661
52774
|
reject(err);
|
|
52662
52775
|
});
|
|
52663
52776
|
hashCalculator.on("error", reject);
|
|
52664
52777
|
hashCalculator.on("finish", () => {
|
|
52665
|
-
hash.digest().then(
|
|
52778
|
+
hash.digest().then(resolve17).catch(reject);
|
|
52666
52779
|
});
|
|
52667
52780
|
});
|
|
52668
52781
|
};
|
|
@@ -57184,8 +57297,8 @@ var init_dist_es22 = __esm({
|
|
|
57184
57297
|
});
|
|
57185
57298
|
|
|
57186
57299
|
// src/cli.ts
|
|
57187
|
-
import { resolve as
|
|
57188
|
-
import { readFileSync as
|
|
57300
|
+
import { resolve as resolve16, dirname as dirname18 } from "path";
|
|
57301
|
+
import { readFileSync as readFileSync25 } from "fs";
|
|
57189
57302
|
import { execSync } from "child_process";
|
|
57190
57303
|
import { parse as parse3 } from "yaml";
|
|
57191
57304
|
|
|
@@ -57453,7 +57566,7 @@ init_http();
|
|
|
57453
57566
|
import { createServer } from "http";
|
|
57454
57567
|
import { spawn as spawn2 } from "child_process";
|
|
57455
57568
|
import { WebSocketServer, WebSocket } from "ws";
|
|
57456
|
-
import { dirname as dirname17, resolve as
|
|
57569
|
+
import { dirname as dirname17, resolve as resolve15 } from "path";
|
|
57457
57570
|
|
|
57458
57571
|
// src/gui/active-db.ts
|
|
57459
57572
|
init_adapter();
|
|
@@ -57461,9 +57574,17 @@ init_members();
|
|
|
57461
57574
|
init_native_entities();
|
|
57462
57575
|
async function changeVisibleToActiveRole(db, payload) {
|
|
57463
57576
|
if (db.getDialect() !== "postgres") return true;
|
|
57464
|
-
if (payload.op === "delete" || payload.op === "DELETE") return true;
|
|
57465
57577
|
if (!payload.table_name || !payload.pk) return false;
|
|
57466
57578
|
try {
|
|
57579
|
+
if (isDeleteOp(payload.op)) {
|
|
57580
|
+
if (payload.del_owner_role == null) return false;
|
|
57581
|
+
const row2 = await getAsyncOrSync(
|
|
57582
|
+
db.adapter,
|
|
57583
|
+
`SELECT lattice_delete_visible(?, ?, ?::text[]) AS v`,
|
|
57584
|
+
[payload.del_owner_role, payload.del_visibility ?? null, payload.del_grantees ?? []]
|
|
57585
|
+
);
|
|
57586
|
+
return row2?.v === true || row2?.v === "t" || row2?.v === 1;
|
|
57587
|
+
}
|
|
57467
57588
|
const row = await getAsyncOrSync(db.adapter, `SELECT lattice_row_visible(?, ?) AS v`, [
|
|
57468
57589
|
payload.table_name,
|
|
57469
57590
|
payload.pk
|
|
@@ -58313,9 +58434,9 @@ var RealtimeBroker = class {
|
|
|
58313
58434
|
() => "ended"
|
|
58314
58435
|
// a graceful-close error is still "closed enough"
|
|
58315
58436
|
);
|
|
58316
|
-
const timedOut = new Promise((
|
|
58437
|
+
const timedOut = new Promise((resolve17) => {
|
|
58317
58438
|
timer = setTimeout(() => {
|
|
58318
|
-
|
|
58439
|
+
resolve17("timeout");
|
|
58319
58440
|
}, this.stopEndTimeoutMs);
|
|
58320
58441
|
timer.unref?.();
|
|
58321
58442
|
});
|
|
@@ -58360,7 +58481,10 @@ function parsePayload(raw) {
|
|
|
58360
58481
|
pk: typeof obj2.pk === "string" ? obj2.pk : null,
|
|
58361
58482
|
op: obj2.op,
|
|
58362
58483
|
owner_role: typeof obj2.owner_role === "string" ? obj2.owner_role : null,
|
|
58363
|
-
created_at: typeof obj2.created_at === "string" ? obj2.created_at : ""
|
|
58484
|
+
created_at: typeof obj2.created_at === "string" ? obj2.created_at : "",
|
|
58485
|
+
del_owner_role: typeof obj2.del_owner_role === "string" ? obj2.del_owner_role : null,
|
|
58486
|
+
del_visibility: typeof obj2.del_visibility === "string" ? obj2.del_visibility : null,
|
|
58487
|
+
del_grantees: Array.isArray(obj2.del_grantees) ? obj2.del_grantees.filter((g6) => typeof g6 === "string") : null
|
|
58364
58488
|
};
|
|
58365
58489
|
} catch {
|
|
58366
58490
|
return null;
|
|
@@ -59341,9 +59465,9 @@ function startBackgroundRender(active) {
|
|
|
59341
59465
|
}
|
|
59342
59466
|
function settleWithin(p3, ms) {
|
|
59343
59467
|
let timer;
|
|
59344
|
-
const timeout = new Promise((
|
|
59468
|
+
const timeout = new Promise((resolve17) => {
|
|
59345
59469
|
timer = setTimeout(() => {
|
|
59346
|
-
|
|
59470
|
+
resolve17("timeout");
|
|
59347
59471
|
}, ms);
|
|
59348
59472
|
timer.unref?.();
|
|
59349
59473
|
});
|
|
@@ -59398,9 +59522,9 @@ var SWITCH_OPEN_TIMEOUT_MS = 2e4;
|
|
|
59398
59522
|
async function openWithinTimeout(open, timeoutMs = SWITCH_OPEN_TIMEOUT_MS, dispose = disposeActive) {
|
|
59399
59523
|
const opening = open();
|
|
59400
59524
|
let timer;
|
|
59401
|
-
const timedOut = new Promise((
|
|
59525
|
+
const timedOut = new Promise((resolve17) => {
|
|
59402
59526
|
timer = setTimeout(() => {
|
|
59403
|
-
|
|
59527
|
+
resolve17("timeout");
|
|
59404
59528
|
}, timeoutMs);
|
|
59405
59529
|
timer.unref?.();
|
|
59406
59530
|
});
|
|
@@ -61071,6 +61195,65 @@ var chatCss = ` /* \u2500\u2500 Chat bubbles + tool pills \u2500\u2500\u2500\
|
|
|
61071
61195
|
}
|
|
61072
61196
|
`;
|
|
61073
61197
|
|
|
61198
|
+
// src/gui/app/styles/inline-import.ts
|
|
61199
|
+
var inlineImportCss = `
|
|
61200
|
+
/* \u2500\u2500 Inline import confirm card (assistant rail) \u2500\u2500 */
|
|
61201
|
+
.cd-sub { margin: 10px 0 6px; font-size: 12px; color: var(--text-muted, #9aa3ad); }
|
|
61202
|
+
.cd-row { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; margin-top: 8px; }
|
|
61203
|
+
.cd-path {
|
|
61204
|
+
flex: 1 1 220px; min-width: 0; box-sizing: border-box; height: 34px; padding: 0 10px;
|
|
61205
|
+
border-radius: 6px; border: 1px solid #2a2f36;
|
|
61206
|
+
background: var(--panel, #0e1116); color: var(--text, #e6e8eb); font-size: 13px;
|
|
61207
|
+
}
|
|
61208
|
+
.cd-status { margin-top: 12px; font-size: 13px; line-height: 1.5; }
|
|
61209
|
+
.cd-status.ok { color: #bef264; }
|
|
61210
|
+
.cd-status.err { color: #f87171; }
|
|
61211
|
+
.cd-status a { color: var(--accent, #bef264); }
|
|
61212
|
+
.cd-btn {
|
|
61213
|
+
height: 34px; padding: 0 14px; border-radius: 6px; border: 1px solid #2a2f36;
|
|
61214
|
+
background: transparent; color: var(--text, #e6e8eb); font-size: 13px;
|
|
61215
|
+
font-weight: 600; cursor: pointer;
|
|
61216
|
+
}
|
|
61217
|
+
.cd-btn:hover { background: rgba(255, 255, 255, 0.06); }
|
|
61218
|
+
.cd-btn.cd-primary { background: #bef264; color: #0b0d10; border-color: #bef264; }
|
|
61219
|
+
.cd-btn.cd-primary:hover { filter: brightness(1.06); }
|
|
61220
|
+
.cd-import-list { margin: 10px 0 0; padding-left: 18px; font-size: 13px; line-height: 1.6; }
|
|
61221
|
+
.cd-import-list li { margin: 2px 0; }
|
|
61222
|
+
.imp-sub { margin: 16px 0 6px; font-size: 13px; color: var(--text, #e6e8eb); }
|
|
61223
|
+
.imp-modes { display: flex; flex-direction: column; gap: 8px; margin: 0 0 6px; }
|
|
61224
|
+
.imp-modes label {
|
|
61225
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
61226
|
+
padding: 8px 10px; border: 1px solid #2a2f36; border-radius: 6px; cursor: pointer;
|
|
61227
|
+
}
|
|
61228
|
+
.imp-modes label:hover { background: rgba(255, 255, 255, 0.04); }
|
|
61229
|
+
.imp-modes input { margin-top: 2px; }
|
|
61230
|
+
.imp-modes b { color: var(--text, #e6e8eb); }
|
|
61231
|
+
.imp-percol {
|
|
61232
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
61233
|
+
margin: 8px 0 0; cursor: pointer; color: var(--text-dim, #aeb6c2);
|
|
61234
|
+
}
|
|
61235
|
+
.imp-percol input { margin-top: 2px; }
|
|
61236
|
+
.imp-match { border-left: 3px solid var(--accent, #7dd3fc); font-weight: 500; }
|
|
61237
|
+
.feed-item.import-confirm .imp-confirm-body { margin-top: 4px; }
|
|
61238
|
+
|
|
61239
|
+
/* \u2500\u2500 Live import progress in the card's log \u2500\u2500 */
|
|
61240
|
+
.feed-item.import-confirm .imp-card-log,
|
|
61241
|
+
.feed-item.import-live .imp-card-log {
|
|
61242
|
+
margin-top: 4px;
|
|
61243
|
+
font: 12px/1.6 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
|
61244
|
+
max-height: 200px; overflow-y: auto; color: var(--text-muted, #9aa3ad);
|
|
61245
|
+
}
|
|
61246
|
+
.imp-card-line { white-space: pre-wrap; word-break: break-word; }
|
|
61247
|
+
.imp-card-line.imp-done { color: var(--accent, #bef264); }
|
|
61248
|
+
.imp-card-line.imp-err { color: #f87171; }
|
|
61249
|
+
.imp-card-line.imp-spin::after {
|
|
61250
|
+
content: ''; display: inline-block; width: 10px; height: 10px; margin-left: 7px;
|
|
61251
|
+
border: 2px solid currentColor; border-right-color: transparent; border-radius: 50%;
|
|
61252
|
+
vertical-align: -1px; animation: imp-spin-kf 0.7s linear infinite;
|
|
61253
|
+
}
|
|
61254
|
+
@keyframes imp-spin-kf { to { transform: rotate(360deg); } }
|
|
61255
|
+
`;
|
|
61256
|
+
|
|
61074
61257
|
// src/gui/app/styles/index.ts
|
|
61075
61258
|
var css = [
|
|
61076
61259
|
tokensCss,
|
|
@@ -61092,7 +61275,8 @@ var css = [
|
|
|
61092
61275
|
fsWorkspaceCss,
|
|
61093
61276
|
settingsDrawerCss,
|
|
61094
61277
|
assistantRailCss,
|
|
61095
|
-
chatCss
|
|
61278
|
+
chatCss,
|
|
61279
|
+
inlineImportCss
|
|
61096
61280
|
].join("");
|
|
61097
61281
|
|
|
61098
61282
|
// src/gui/app/modules/display-config.ts
|
|
@@ -68663,6 +68847,11 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
68663
68847
|
// survivor if it was a duplicate). Multi-file drops do not navigate.
|
|
68664
68848
|
if (files.length === 1) {
|
|
68665
68849
|
uploadFile(files[0]).then(function (j) {
|
|
68850
|
+
// A structured source the server flagged as confirmable comes back with
|
|
68851
|
+
// an autoImport proposal \u2014 render the inline confirm card instead of
|
|
68852
|
+
// navigating to the file record. A silent import (autoImport.imported,
|
|
68853
|
+
// no reason) or a plain file keeps the open-the-record behavior.
|
|
68854
|
+
if (j && j.autoImport && j.autoImport.reason) { renderInlineImportCard(j.autoImport); return; }
|
|
68666
68855
|
if (j && (j.duplicateOf || j.id)) openSearchHit('files', j.duplicateOf || j.id);
|
|
68667
68856
|
});
|
|
68668
68857
|
return;
|
|
@@ -68672,7 +68861,15 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
68672
68861
|
var bar = ingestProgress(files.length);
|
|
68673
68862
|
var thunks = [];
|
|
68674
68863
|
for (var i = 0; i < files.length; i++) {
|
|
68675
|
-
(function (f) {
|
|
68864
|
+
(function (f) {
|
|
68865
|
+
thunks.push(function () {
|
|
68866
|
+
return uploadFile(f).then(function (j) {
|
|
68867
|
+
// A structured source within a batch still gets its own inline
|
|
68868
|
+
// confirm card (the batch as a whole does not navigate).
|
|
68869
|
+
if (j && j.autoImport && j.autoImport.reason) renderInlineImportCard(j.autoImport);
|
|
68870
|
+
});
|
|
68871
|
+
});
|
|
68872
|
+
})(files[i]);
|
|
68676
68873
|
}
|
|
68677
68874
|
runIngestBatch(thunks, INGEST_MAX_CONCURRENCY, bar.update).then(bar.done);
|
|
68678
68875
|
}
|
|
@@ -68828,6 +69025,237 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
68828
69025
|
})();
|
|
68829
69026
|
`;
|
|
68830
69027
|
|
|
69028
|
+
// src/gui/app/modules/inline-import.ts
|
|
69029
|
+
var inlineImportJs = `
|
|
69030
|
+
// \u2500\u2500 Inline structured-source import (confirm card in the assistant rail) \u2500\u2500
|
|
69031
|
+
function iiRailFeed() { return document.getElementById('rail-feed'); }
|
|
69032
|
+
function iiRailEmptyGone() {
|
|
69033
|
+
var e = document.getElementById('rail-empty');
|
|
69034
|
+
if (e) e.parentNode && e.parentNode.removeChild(e);
|
|
69035
|
+
}
|
|
69036
|
+
|
|
69037
|
+
// Read a newline-delimited-JSON response body, invoking onEvent(obj) per line.
|
|
69038
|
+
// Self-contained on purpose \u2014 this segment must not depend on any other.
|
|
69039
|
+
function iiStreamNdjson(url, payload, onEvent) {
|
|
69040
|
+
fetch(url, {
|
|
69041
|
+
method: 'POST',
|
|
69042
|
+
headers: { 'content-type': 'application/json' },
|
|
69043
|
+
body: JSON.stringify(payload),
|
|
69044
|
+
}).then(function (res) {
|
|
69045
|
+
if (!res.body || !res.body.getReader) {
|
|
69046
|
+
return res.text().then(function (t) {
|
|
69047
|
+
t.split('\\n').forEach(function (line) {
|
|
69048
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
69049
|
+
});
|
|
69050
|
+
});
|
|
69051
|
+
}
|
|
69052
|
+
var reader = res.body.getReader();
|
|
69053
|
+
var dec = new TextDecoder();
|
|
69054
|
+
var buf = '';
|
|
69055
|
+
function pump() {
|
|
69056
|
+
return reader.read().then(function (chunk) {
|
|
69057
|
+
if (chunk.done) {
|
|
69058
|
+
if (buf.trim()) { try { onEvent(JSON.parse(buf)); } catch (e) { /* skip */ } }
|
|
69059
|
+
return;
|
|
69060
|
+
}
|
|
69061
|
+
buf += dec.decode(chunk.value, { stream: true });
|
|
69062
|
+
var idx;
|
|
69063
|
+
while ((idx = buf.indexOf('\\n')) >= 0) {
|
|
69064
|
+
var line = buf.slice(0, idx);
|
|
69065
|
+
buf = buf.slice(idx + 1);
|
|
69066
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
69067
|
+
}
|
|
69068
|
+
return pump();
|
|
69069
|
+
});
|
|
69070
|
+
}
|
|
69071
|
+
return pump();
|
|
69072
|
+
}).catch(function (err) {
|
|
69073
|
+
onEvent({ phase: 'error', message: err && err.message ? err.message : 'Request failed' });
|
|
69074
|
+
});
|
|
69075
|
+
}
|
|
69076
|
+
|
|
69077
|
+
// Render the confirm card for a structured drop the server flagged as
|
|
69078
|
+
// needing confirmation. autoImport is the upload response's proposal:
|
|
69079
|
+
// { reason, fileId, plan:{entities,dimensions,linkages}, views, asOf,
|
|
69080
|
+
// asOfCandidates, asOfColumns, schemaMatch, matchedCount, totalEntities }.
|
|
69081
|
+
function renderInlineImportCard(autoImport) {
|
|
69082
|
+
if (!autoImport || !autoImport.fileId) return;
|
|
69083
|
+
var plan = autoImport.plan || {};
|
|
69084
|
+
var ents = plan.entities || [];
|
|
69085
|
+
var dims = plan.dimensions || [];
|
|
69086
|
+
var links = plan.linkages || [];
|
|
69087
|
+
var views = autoImport.views || [];
|
|
69088
|
+
var candidates = autoImport.asOfCandidates || [];
|
|
69089
|
+
var asOfColumns = autoImport.asOfColumns || [];
|
|
69090
|
+
var schemaMatch = autoImport.schemaMatch || {};
|
|
69091
|
+
var headerText = autoImport.reason === 'needs-confirm'
|
|
69092
|
+
? 'Add a dated snapshot'
|
|
69093
|
+
: 'Import as a new dataset';
|
|
69094
|
+
|
|
69095
|
+
iiRailEmptyGone();
|
|
69096
|
+
var feedEl = iiRailFeed();
|
|
69097
|
+
var card = document.createElement('div');
|
|
69098
|
+
card.className = 'feed-item import-confirm';
|
|
69099
|
+
var icon = document.createElement('div');
|
|
69100
|
+
icon.className = 'feed-icon';
|
|
69101
|
+
icon.textContent = '\u2913';
|
|
69102
|
+
var bodyEl = document.createElement('div');
|
|
69103
|
+
bodyEl.className = 'feed-body';
|
|
69104
|
+
var title = document.createElement('div');
|
|
69105
|
+
title.className = 'feed-summary';
|
|
69106
|
+
title.textContent = headerText;
|
|
69107
|
+
bodyEl.appendChild(title);
|
|
69108
|
+
|
|
69109
|
+
var parts = [];
|
|
69110
|
+
if (schemaMatch.isKnownDocument) {
|
|
69111
|
+
parts.push('<div class="cd-status ok imp-match">Recognized as a new period of an existing document — ' +
|
|
69112
|
+
schemaMatch.matchedCount + ' of ' + schemaMatch.totalEntities +
|
|
69113
|
+
' tables match what you already imported. It will be added as a dated snapshot.</div>');
|
|
69114
|
+
}
|
|
69115
|
+
parts.push('<div class="cd-status ok">Found ' + ents.length + ' entities, ' + dims.length +
|
|
69116
|
+
' dimensions, ' + links.length + ' links' +
|
|
69117
|
+
(views.length ? ', ' + views.length + ' reconstructed views (no duplicated rows)' : '') +
|
|
69118
|
+
'.</div><ul class="cd-import-list">');
|
|
69119
|
+
ents.forEach(function (e) {
|
|
69120
|
+
parts.push('<li><b>' + escapeHtml(e.name) + '</b> — ' + e.rowCount + ' rows, ' +
|
|
69121
|
+
(e.columns ? e.columns.length : 0) + ' cols · ' +
|
|
69122
|
+
(e.naturalKey ? 'key ' + escapeHtml(e.naturalKey) : 'keyless') + '</li>');
|
|
69123
|
+
});
|
|
69124
|
+
dims.forEach(function (d) {
|
|
69125
|
+
parts.push('<li><b>' + escapeHtml(d.name) + '</b> (dimension) — ' + d.distinctValues + ' values</li>');
|
|
69126
|
+
});
|
|
69127
|
+
views.forEach(function (v) {
|
|
69128
|
+
parts.push('<li><b>' + escapeHtml(v.name) + '</b> (view of ' + escapeHtml(v.master) + ' where ' +
|
|
69129
|
+
escapeHtml(v.filterColumn) + ' = ' + escapeHtml(String(v.filterValue)) + ') — ' +
|
|
69130
|
+
v.matchedRows + ' rows, not duplicated</li>');
|
|
69131
|
+
});
|
|
69132
|
+
parts.push('</ul>');
|
|
69133
|
+
|
|
69134
|
+
parts.push('<h4 class="imp-sub">As of date</h4>');
|
|
69135
|
+
var best = candidates[0];
|
|
69136
|
+
parts.push('<p class="cd-sub">' +
|
|
69137
|
+
(best ? 'Detected from ' + escapeHtml(best.evidence) + ' — edit if wrong.'
|
|
69138
|
+
: 'No date found in the file or its name — set the snapshot date, or leave blank to import undated.') +
|
|
69139
|
+
' A newer file is kept as a separate dated snapshot beside the prior one.</p>');
|
|
69140
|
+
parts.push('<div class="cd-row"><input class="cd-path" id="ii-asof" type="date" value="' + escapeHtml(autoImport.asOf || '') + '" aria-label="As of date" /></div>');
|
|
69141
|
+
if (candidates.length > 1) {
|
|
69142
|
+
parts.push('<div class="cd-sub">Other candidates: ' + candidates.slice(1, 5).map(function (c) {
|
|
69143
|
+
return '<a href="#" class="ii-asof-alt" data-date="' + escapeHtml(c.date) + '" title="' + escapeHtml(c.evidence) + '">' + escapeHtml(c.date) + '</a>';
|
|
69144
|
+
}).join(', ') + '</div>');
|
|
69145
|
+
}
|
|
69146
|
+
if (asOfColumns.length) {
|
|
69147
|
+
var colOpts = asOfColumns.slice(0, 6).map(function (c) {
|
|
69148
|
+
return '<option value="' + escapeHtml(c.column) + '" title="' + escapeHtml(c.evidence) + '">' +
|
|
69149
|
+
escapeHtml(c.column) + ' (' + escapeHtml(c.entity) + ', ' + c.distinctDates +
|
|
69150
|
+
' date' + (c.distinctDates === 1 ? '' : 's') + ')</option>';
|
|
69151
|
+
}).join('');
|
|
69152
|
+
parts.push('<label class="imp-percol"><input type="checkbox" id="ii-asof-percol"> ' +
|
|
69153
|
+
'<span>Date varies per row — use a date column instead (one file, many periods)</span></label>');
|
|
69154
|
+
parts.push('<div class="cd-row" id="ii-asof-col-row" style="display:none"><select class="cd-path" id="ii-asof-col">' + colOpts + '</select></div>');
|
|
69155
|
+
}
|
|
69156
|
+
|
|
69157
|
+
parts.push('<h4 class="imp-sub">What should Lattice bring in?</h4>');
|
|
69158
|
+
parts.push('<div class="imp-modes">' +
|
|
69159
|
+
'<label><input type="radio" name="ii-mode" value="both" checked> <span><b>Data model + contents</b> \u2014 the schema, the taxonomy, and all the rows.</span></label>' +
|
|
69160
|
+
'<label><input type="radio" name="ii-mode" value="schema"> <span><b>Data model / schema only</b> \u2014 tables, dimension values, and views. No rows.</span></label>' +
|
|
69161
|
+
'<label><input type="radio" name="ii-mode" value="contents"> <span><b>Contents only</b> \u2014 the rows and their links, into tables that already exist.</span></label>' +
|
|
69162
|
+
'</div>');
|
|
69163
|
+
parts.push('<div class="cd-row"><button class="cd-btn cd-primary" id="ii-apply" type="button">Import into Lattice</button></div>');
|
|
69164
|
+
parts.push('<div class="imp-card-log" id="ii-log"></div>');
|
|
69165
|
+
|
|
69166
|
+
var content = document.createElement('div');
|
|
69167
|
+
content.className = 'imp-confirm-body';
|
|
69168
|
+
content.innerHTML = parts.join('');
|
|
69169
|
+
bodyEl.appendChild(content);
|
|
69170
|
+
card.appendChild(icon);
|
|
69171
|
+
card.appendChild(bodyEl);
|
|
69172
|
+
if (feedEl) { feedEl.appendChild(card); feedEl.scrollTop = feedEl.scrollHeight; }
|
|
69173
|
+
|
|
69174
|
+
content.querySelectorAll('.ii-asof-alt').forEach(function (a) {
|
|
69175
|
+
a.addEventListener('click', function (e) {
|
|
69176
|
+
e.preventDefault();
|
|
69177
|
+
var input = document.getElementById('ii-asof');
|
|
69178
|
+
if (input) input.value = a.getAttribute('data-date') || '';
|
|
69179
|
+
});
|
|
69180
|
+
});
|
|
69181
|
+
var perCol = document.getElementById('ii-asof-percol');
|
|
69182
|
+
if (perCol) perCol.addEventListener('change', function () {
|
|
69183
|
+
var row = document.getElementById('ii-asof-col-row');
|
|
69184
|
+
var dateEl = document.getElementById('ii-asof');
|
|
69185
|
+
if (row) row.style.display = perCol.checked ? '' : 'none';
|
|
69186
|
+
if (dateEl) dateEl.disabled = perCol.checked;
|
|
69187
|
+
});
|
|
69188
|
+
|
|
69189
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
69190
|
+
if (applyBtn) applyBtn.addEventListener('click', function () {
|
|
69191
|
+
runInlineImport(autoImport.fileId, title, content);
|
|
69192
|
+
});
|
|
69193
|
+
}
|
|
69194
|
+
|
|
69195
|
+
// POST the confirmed proposal to /api/import/apply and stream the pipeline
|
|
69196
|
+
// live into the card's log. On 'done' show a success summary + refresh the
|
|
69197
|
+
// Objects nav in place; on 'error' show the message.
|
|
69198
|
+
function runInlineImport(fileId, title, content) {
|
|
69199
|
+
var sel = content.querySelector('input[name="ii-mode"]:checked');
|
|
69200
|
+
var mode = sel ? sel.value : 'both';
|
|
69201
|
+
var asofEl = document.getElementById('ii-asof');
|
|
69202
|
+
var asOf = asofEl ? asofEl.value : '';
|
|
69203
|
+
var perColEl = document.getElementById('ii-asof-percol');
|
|
69204
|
+
var colSel = document.getElementById('ii-asof-col');
|
|
69205
|
+
var asOfColumn = (perColEl && perColEl.checked && colSel) ? colSel.value : '';
|
|
69206
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
69207
|
+
if (applyBtn) applyBtn.disabled = true;
|
|
69208
|
+
|
|
69209
|
+
var feedEl = iiRailFeed();
|
|
69210
|
+
var log = document.getElementById('ii-log');
|
|
69211
|
+
function addLine(text, cls) {
|
|
69212
|
+
if (!log) return null;
|
|
69213
|
+
var d = document.createElement('div');
|
|
69214
|
+
d.className = 'imp-card-line' + (cls ? ' ' + cls : '');
|
|
69215
|
+
d.textContent = text;
|
|
69216
|
+
log.appendChild(d);
|
|
69217
|
+
while (log.childNodes.length > 60) log.removeChild(log.firstChild);
|
|
69218
|
+
log.scrollTop = log.scrollHeight;
|
|
69219
|
+
if (feedEl) feedEl.scrollTop = feedEl.scrollHeight;
|
|
69220
|
+
return d;
|
|
69221
|
+
}
|
|
69222
|
+
title.textContent = 'Importing your data\u2026';
|
|
69223
|
+
addLine('Starting\u2026');
|
|
69224
|
+
|
|
69225
|
+
iiStreamNdjson('/api/import/apply', { fileId: fileId, mode: mode, asOf: asOf, asOfColumn: asOfColumn }, function (evt) {
|
|
69226
|
+
if (!evt) return;
|
|
69227
|
+
if (evt.phase === 'done') {
|
|
69228
|
+
var r = evt.result || {};
|
|
69229
|
+
var rbt = r.rowsByTable || {};
|
|
69230
|
+
var names = Object.keys(rbt);
|
|
69231
|
+
var total = 0;
|
|
69232
|
+
names.forEach(function (n) { total += (rbt[n] || 0); });
|
|
69233
|
+
title.textContent = 'Imported ' + names.length + ' tables' + (mode === 'schema' ? '' : ', ' + total + ' rows');
|
|
69234
|
+
var upd = addLine('Updating your objects\u2026', 'imp-spin');
|
|
69235
|
+
refreshEntities().then(function () {
|
|
69236
|
+
renderSidebar();
|
|
69237
|
+
renderRoute();
|
|
69238
|
+
var count = (state.entities && state.entities.tables) ? state.entities.tables.length : names.length;
|
|
69239
|
+
if (upd) {
|
|
69240
|
+
upd.className = 'imp-card-line imp-done';
|
|
69241
|
+
upd.textContent = '\u2713 Done \u2014 ' + count + ' objects in your workspace';
|
|
69242
|
+
}
|
|
69243
|
+
}).catch(function () {
|
|
69244
|
+
if (upd) {
|
|
69245
|
+
upd.className = 'imp-card-line imp-err';
|
|
69246
|
+
upd.textContent = 'Imported, but refreshing the view failed \u2014 reload to see your objects.';
|
|
69247
|
+
}
|
|
69248
|
+
});
|
|
69249
|
+
} else if (evt.phase === 'error') {
|
|
69250
|
+
title.textContent = 'Import failed';
|
|
69251
|
+
addLine('Error: ' + (evt.message || 'import failed'), 'imp-err');
|
|
69252
|
+
} else if (evt.message) {
|
|
69253
|
+
addLine(evt.message);
|
|
69254
|
+
}
|
|
69255
|
+
});
|
|
69256
|
+
}
|
|
69257
|
+
`;
|
|
69258
|
+
|
|
68831
69259
|
// src/gui/app/modules/index.ts
|
|
68832
69260
|
var appJs = [
|
|
68833
69261
|
displayConfigJs,
|
|
@@ -68856,7 +69284,8 @@ var appJs = [
|
|
|
68856
69284
|
dataModelJs,
|
|
68857
69285
|
latticeTeamsJs,
|
|
68858
69286
|
onboardingJs,
|
|
68859
|
-
createDatabaseWizardJs
|
|
69287
|
+
createDatabaseWizardJs,
|
|
69288
|
+
inlineImportJs
|
|
68860
69289
|
].join("");
|
|
68861
69290
|
|
|
68862
69291
|
// src/gui/app/analytics.ts
|
|
@@ -71432,7 +71861,7 @@ init_extract();
|
|
|
71432
71861
|
import { statSync as statSync8 } from "fs";
|
|
71433
71862
|
import { writeFile as writeFile2, rm } from "fs/promises";
|
|
71434
71863
|
import { tmpdir as tmpdir2 } from "os";
|
|
71435
|
-
import { basename as
|
|
71864
|
+
import { basename as basename11, extname as extname2, resolve as resolve11, join as join28 } from "path";
|
|
71436
71865
|
|
|
71437
71866
|
// src/ai/vision.ts
|
|
71438
71867
|
init_llm_client();
|
|
@@ -71555,13 +71984,13 @@ import { createHash as createHash10 } from "crypto";
|
|
|
71555
71984
|
import { createReadStream as createReadStream2, existsSync as existsSync25, mkdirSync as mkdirSync11, statSync as statSync7, copyFileSync as copyFileSync4 } from "fs";
|
|
71556
71985
|
import { basename as basename9, join as join27 } from "path";
|
|
71557
71986
|
async function hashFile(srcPath) {
|
|
71558
|
-
return new Promise((
|
|
71987
|
+
return new Promise((resolve17, reject) => {
|
|
71559
71988
|
const hash = createHash10("sha256");
|
|
71560
71989
|
const stream = createReadStream2(srcPath);
|
|
71561
71990
|
stream.on("data", (chunk) => hash.update(chunk));
|
|
71562
71991
|
stream.on("error", reject);
|
|
71563
71992
|
stream.on("end", () => {
|
|
71564
|
-
|
|
71993
|
+
resolve17(hash.digest("hex"));
|
|
71565
71994
|
});
|
|
71566
71995
|
});
|
|
71567
71996
|
}
|
|
@@ -71593,8 +72022,1158 @@ init_http();
|
|
|
71593
72022
|
init_enrich();
|
|
71594
72023
|
init_ingest_url();
|
|
71595
72024
|
init_file_row();
|
|
71596
|
-
import { createHash as
|
|
72025
|
+
import { createHash as createHash12 } from "crypto";
|
|
71597
72026
|
init_dedup_service();
|
|
72027
|
+
|
|
72028
|
+
// src/gui/import-auto.ts
|
|
72029
|
+
import { readFileSync as readFileSync22 } from "fs";
|
|
72030
|
+
|
|
72031
|
+
// src/import/infer.ts
|
|
72032
|
+
var SAMPLE = 300;
|
|
72033
|
+
var PREFERRED_KEYS = ["code", "id", "slug", "key", "ticker", "symbol"];
|
|
72034
|
+
var NEVER_KEY = /* @__PURE__ */ new Set([
|
|
72035
|
+
"description",
|
|
72036
|
+
"notes",
|
|
72037
|
+
"summary",
|
|
72038
|
+
"desc",
|
|
72039
|
+
"comment",
|
|
72040
|
+
"comments",
|
|
72041
|
+
"bio",
|
|
72042
|
+
"text",
|
|
72043
|
+
"body"
|
|
72044
|
+
]);
|
|
72045
|
+
var FREETEXT = /* @__PURE__ */ new Set([...NEVER_KEY, "name", "title", "company", "label"]);
|
|
72046
|
+
var DIM_MAX_DISTINCT = 64;
|
|
72047
|
+
var DIM_MAX_RATIO = 0.5;
|
|
72048
|
+
var LINK_MIN_CONFIDENCE = 0.3;
|
|
72049
|
+
function isPlainObject(v2) {
|
|
72050
|
+
return typeof v2 === "object" && v2 !== null && !Array.isArray(v2);
|
|
72051
|
+
}
|
|
72052
|
+
function sourceRecords(data, entity) {
|
|
72053
|
+
const v2 = data[entity.sourceKey];
|
|
72054
|
+
if (!Array.isArray(v2)) return [];
|
|
72055
|
+
if (entity.columnar) {
|
|
72056
|
+
const cols = data[entity.sourceKey + "Cols"];
|
|
72057
|
+
if (!Array.isArray(cols)) return [];
|
|
72058
|
+
return v2.map((row) => {
|
|
72059
|
+
const o3 = {};
|
|
72060
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
72061
|
+
return o3;
|
|
72062
|
+
});
|
|
72063
|
+
}
|
|
72064
|
+
return v2.filter(isPlainObject);
|
|
72065
|
+
}
|
|
72066
|
+
function normalizeName(key) {
|
|
72067
|
+
const s2 = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
72068
|
+
if (!s2) return "field";
|
|
72069
|
+
return /^[a-z]/.test(s2) ? s2 : "f_" + s2;
|
|
72070
|
+
}
|
|
72071
|
+
var ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
72072
|
+
var ISO_DATETIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/;
|
|
72073
|
+
function inferFieldType(values) {
|
|
72074
|
+
const present = values.filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
72075
|
+
if (present.length === 0) return "text";
|
|
72076
|
+
if (present.every((v2) => typeof v2 === "number")) {
|
|
72077
|
+
return present.every((v2) => Number.isInteger(v2)) ? "integer" : "real";
|
|
72078
|
+
}
|
|
72079
|
+
if (present.every((v2) => typeof v2 === "boolean")) return "boolean";
|
|
72080
|
+
if (present.every((v2) => typeof v2 === "string")) {
|
|
72081
|
+
if (present.every((v2) => ISO_DATE.test(v2))) return "date";
|
|
72082
|
+
if (present.every((v2) => ISO_DATETIME.test(v2))) return "datetime";
|
|
72083
|
+
}
|
|
72084
|
+
return "text";
|
|
72085
|
+
}
|
|
72086
|
+
function norm2(v2) {
|
|
72087
|
+
return String(v2).trim().toLowerCase();
|
|
72088
|
+
}
|
|
72089
|
+
function isNumericValue(v2) {
|
|
72090
|
+
if (typeof v2 === "number") return Number.isFinite(v2);
|
|
72091
|
+
if (typeof v2 !== "string") return false;
|
|
72092
|
+
const s2 = v2.replace(/[\s,$%()]/g, "");
|
|
72093
|
+
return s2 !== "" && Number.isFinite(Number(s2));
|
|
72094
|
+
}
|
|
72095
|
+
function profileColumns(records) {
|
|
72096
|
+
const keys = /* @__PURE__ */ new Set();
|
|
72097
|
+
for (const r6 of records.slice(0, SAMPLE)) for (const k6 of Object.keys(r6)) keys.add(k6);
|
|
72098
|
+
const out = /* @__PURE__ */ new Map();
|
|
72099
|
+
for (const key of keys) {
|
|
72100
|
+
let isArray = false;
|
|
72101
|
+
const sample = [];
|
|
72102
|
+
const valueSet = /* @__PURE__ */ new Set();
|
|
72103
|
+
const distinctSet = /* @__PURE__ */ new Set();
|
|
72104
|
+
let nonNull = 0;
|
|
72105
|
+
let numeric = 0;
|
|
72106
|
+
for (const r6 of records) {
|
|
72107
|
+
const v2 = r6[key];
|
|
72108
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
72109
|
+
nonNull++;
|
|
72110
|
+
if (Array.isArray(v2)) {
|
|
72111
|
+
isArray = true;
|
|
72112
|
+
for (const e6 of v2) {
|
|
72113
|
+
if (e6 !== null && e6 !== void 0 && e6 !== "") {
|
|
72114
|
+
valueSet.add(norm2(e6));
|
|
72115
|
+
distinctSet.add(norm2(e6));
|
|
72116
|
+
}
|
|
72117
|
+
}
|
|
72118
|
+
} else {
|
|
72119
|
+
if (sample.length < SAMPLE) sample.push(v2);
|
|
72120
|
+
if (typeof v2 === "string") valueSet.add(norm2(v2));
|
|
72121
|
+
distinctSet.add(norm2(v2));
|
|
72122
|
+
if (isNumericValue(v2)) numeric++;
|
|
72123
|
+
}
|
|
72124
|
+
}
|
|
72125
|
+
out.set(key, {
|
|
72126
|
+
sourceKey: key,
|
|
72127
|
+
isArray,
|
|
72128
|
+
type: isArray ? "text" : inferFieldType(sample),
|
|
72129
|
+
// Cardinality counts ALL distinct values (numbers + strings). Counting only
|
|
72130
|
+
// string values let a mostly-numeric column with a few text sentinels (e.g.
|
|
72131
|
+
// a "TEV/EBITDA" of numbers + "NM") look low-cardinality and slip in as a
|
|
72132
|
+
// junk dimension.
|
|
72133
|
+
distinct: distinctSet.size,
|
|
72134
|
+
valueSet,
|
|
72135
|
+
numericFraction: nonNull > 0 ? numeric / nonNull : 0
|
|
72136
|
+
});
|
|
72137
|
+
}
|
|
72138
|
+
return out;
|
|
72139
|
+
}
|
|
72140
|
+
function pickNaturalKey(records, profiles) {
|
|
72141
|
+
const n3 = records.length;
|
|
72142
|
+
const isUnique = (key) => {
|
|
72143
|
+
const seen = /* @__PURE__ */ new Set();
|
|
72144
|
+
for (const r6 of records) {
|
|
72145
|
+
const v2 = r6[key];
|
|
72146
|
+
if (v2 === null || v2 === void 0 || v2 === "") return false;
|
|
72147
|
+
const k6 = norm2(v2);
|
|
72148
|
+
if (seen.has(k6)) return false;
|
|
72149
|
+
seen.add(k6);
|
|
72150
|
+
}
|
|
72151
|
+
return seen.size === n3;
|
|
72152
|
+
};
|
|
72153
|
+
for (const pref of PREFERRED_KEYS) {
|
|
72154
|
+
for (const [key, p3] of profiles) {
|
|
72155
|
+
if (p3.isArray) continue;
|
|
72156
|
+
if (normalizeName(key) === pref && isUnique(key)) return key;
|
|
72157
|
+
}
|
|
72158
|
+
}
|
|
72159
|
+
for (const [key, p3] of profiles) {
|
|
72160
|
+
if (p3.isArray) continue;
|
|
72161
|
+
if (NEVER_KEY.has(normalizeName(key))) continue;
|
|
72162
|
+
if ((p3.type === "text" || p3.type === "integer") && isUnique(key)) return key;
|
|
72163
|
+
}
|
|
72164
|
+
return null;
|
|
72165
|
+
}
|
|
72166
|
+
function inferSchema(data, opts = {}) {
|
|
72167
|
+
const skipped = [];
|
|
72168
|
+
const consumedColsKeys = /* @__PURE__ */ new Set();
|
|
72169
|
+
for (const key of Object.keys(data)) {
|
|
72170
|
+
const v2 = data[key];
|
|
72171
|
+
const cols = data[key + "Cols"];
|
|
72172
|
+
if (Array.isArray(v2) && v2.length > 0 && Array.isArray(v2[0]) && Array.isArray(cols) && cols.every((c6) => typeof c6 === "string")) {
|
|
72173
|
+
consumedColsKeys.add(key + "Cols");
|
|
72174
|
+
}
|
|
72175
|
+
}
|
|
72176
|
+
const sources = [];
|
|
72177
|
+
for (const key of Object.keys(data)) {
|
|
72178
|
+
if (consumedColsKeys.has(key)) continue;
|
|
72179
|
+
const v2 = data[key];
|
|
72180
|
+
if (!Array.isArray(v2) || v2.length === 0) {
|
|
72181
|
+
skipped.push({
|
|
72182
|
+
key,
|
|
72183
|
+
reason: isPlainObject(v2) ? "object (derived/rollup)" : "scalar/empty (meta or derived)"
|
|
72184
|
+
});
|
|
72185
|
+
continue;
|
|
72186
|
+
}
|
|
72187
|
+
let records;
|
|
72188
|
+
let columnar = false;
|
|
72189
|
+
if (isPlainObject(v2[0])) {
|
|
72190
|
+
records = v2.filter(isPlainObject);
|
|
72191
|
+
} else if (Array.isArray(v2[0]) && Array.isArray(data[key + "Cols"])) {
|
|
72192
|
+
const cols = data[key + "Cols"];
|
|
72193
|
+
records = v2.map((row) => {
|
|
72194
|
+
const o3 = {};
|
|
72195
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
72196
|
+
return o3;
|
|
72197
|
+
});
|
|
72198
|
+
columnar = true;
|
|
72199
|
+
} else {
|
|
72200
|
+
skipped.push({ key, reason: "array of scalars (not a record set)" });
|
|
72201
|
+
continue;
|
|
72202
|
+
}
|
|
72203
|
+
const name = opts.rename?.[key] ?? normalizeName(key);
|
|
72204
|
+
const profiles = profileColumns(records);
|
|
72205
|
+
sources.push({
|
|
72206
|
+
name,
|
|
72207
|
+
sourceKey: key,
|
|
72208
|
+
records,
|
|
72209
|
+
columnar,
|
|
72210
|
+
profiles,
|
|
72211
|
+
naturalKey: pickNaturalKey(records, profiles)
|
|
72212
|
+
});
|
|
72213
|
+
}
|
|
72214
|
+
const linkages = [];
|
|
72215
|
+
const consumedFields = /* @__PURE__ */ new Map();
|
|
72216
|
+
const linkedTargets = /* @__PURE__ */ new Map();
|
|
72217
|
+
const consume = (e6, f6) => {
|
|
72218
|
+
let set = consumedFields.get(e6);
|
|
72219
|
+
if (!set) {
|
|
72220
|
+
set = /* @__PURE__ */ new Set();
|
|
72221
|
+
consumedFields.set(e6, set);
|
|
72222
|
+
}
|
|
72223
|
+
set.add(f6);
|
|
72224
|
+
};
|
|
72225
|
+
const markTarget = (e6, t8) => {
|
|
72226
|
+
let set = linkedTargets.get(e6);
|
|
72227
|
+
if (!set) {
|
|
72228
|
+
set = /* @__PURE__ */ new Set();
|
|
72229
|
+
linkedTargets.set(e6, set);
|
|
72230
|
+
}
|
|
72231
|
+
set.add(t8);
|
|
72232
|
+
};
|
|
72233
|
+
function bestTarget(self, values) {
|
|
72234
|
+
if (values.size === 0) return null;
|
|
72235
|
+
let best = null;
|
|
72236
|
+
for (const t8 of sources) {
|
|
72237
|
+
if (t8.name === self.name || !t8.naturalKey) continue;
|
|
72238
|
+
const p3 = t8.profiles.get(t8.naturalKey);
|
|
72239
|
+
if (!p3 || p3.valueSet.size === 0) continue;
|
|
72240
|
+
let matched = 0;
|
|
72241
|
+
for (const v2 of values) if (p3.valueSet.has(v2)) matched++;
|
|
72242
|
+
if (matched > 0 && (best === null || matched > best.matched)) {
|
|
72243
|
+
best = { target: t8, column: t8.naturalKey, matched };
|
|
72244
|
+
}
|
|
72245
|
+
}
|
|
72246
|
+
return best;
|
|
72247
|
+
}
|
|
72248
|
+
for (const pass of ["array", "scalar"]) {
|
|
72249
|
+
for (const e6 of sources) {
|
|
72250
|
+
for (const [field, p3] of e6.profiles) {
|
|
72251
|
+
if (pass === "array" ? !p3.isArray : p3.isArray) continue;
|
|
72252
|
+
if (pass === "scalar") {
|
|
72253
|
+
if (field === e6.naturalKey) continue;
|
|
72254
|
+
if (FREETEXT.has(normalizeName(field)) || NEVER_KEY.has(normalizeName(field))) continue;
|
|
72255
|
+
if (p3.type !== "text") continue;
|
|
72256
|
+
}
|
|
72257
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
72258
|
+
const best = bestTarget(e6, p3.valueSet);
|
|
72259
|
+
if (!best) continue;
|
|
72260
|
+
const confidence = best.matched / p3.valueSet.size;
|
|
72261
|
+
if (confidence < LINK_MIN_CONFIDENCE) continue;
|
|
72262
|
+
if (linkedTargets.get(e6.name)?.has(best.target.name)) {
|
|
72263
|
+
consume(e6.name, field);
|
|
72264
|
+
continue;
|
|
72265
|
+
}
|
|
72266
|
+
const link = {
|
|
72267
|
+
kind: pass === "array" ? "many-to-many" : "many-to-one",
|
|
72268
|
+
fromEntity: e6.name,
|
|
72269
|
+
fromField: field,
|
|
72270
|
+
toEntity: best.target.name,
|
|
72271
|
+
toKey: normalizeName(best.column),
|
|
72272
|
+
matched: best.matched,
|
|
72273
|
+
unresolved: p3.valueSet.size - best.matched,
|
|
72274
|
+
confidence
|
|
72275
|
+
};
|
|
72276
|
+
if (pass === "array") link.junction = `${e6.name}_${best.target.name}`;
|
|
72277
|
+
linkages.push(link);
|
|
72278
|
+
consume(e6.name, field);
|
|
72279
|
+
markTarget(e6.name, best.target.name);
|
|
72280
|
+
}
|
|
72281
|
+
}
|
|
72282
|
+
}
|
|
72283
|
+
const dimColumnNames = /* @__PURE__ */ new Map();
|
|
72284
|
+
for (const e6 of sources) {
|
|
72285
|
+
for (const [field, p3] of e6.profiles) {
|
|
72286
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
72287
|
+
const nn = normalizeName(field);
|
|
72288
|
+
let arr = dimColumnNames.get(nn);
|
|
72289
|
+
if (!arr) {
|
|
72290
|
+
arr = [];
|
|
72291
|
+
dimColumnNames.set(nn, arr);
|
|
72292
|
+
}
|
|
72293
|
+
arr.push(e6);
|
|
72294
|
+
}
|
|
72295
|
+
}
|
|
72296
|
+
const dimensions = [];
|
|
72297
|
+
const dimByName = /* @__PURE__ */ new Map();
|
|
72298
|
+
for (const e6 of sources) {
|
|
72299
|
+
for (const [field, p3] of e6.profiles) {
|
|
72300
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
72301
|
+
if (field === e6.naturalKey) continue;
|
|
72302
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
72303
|
+
const nn = normalizeName(field);
|
|
72304
|
+
if (FREETEXT.has(nn)) continue;
|
|
72305
|
+
const ratio = p3.distinct / Math.max(1, e6.records.length);
|
|
72306
|
+
const sharedAcross = dimColumnNames.get(nn)?.length ?? 1;
|
|
72307
|
+
const isDim = p3.distinct >= 1 && p3.distinct <= DIM_MAX_DISTINCT && (ratio <= DIM_MAX_RATIO || sharedAcross >= 2);
|
|
72308
|
+
if (!isDim) continue;
|
|
72309
|
+
let dim = dimByName.get(nn);
|
|
72310
|
+
if (!dim) {
|
|
72311
|
+
dim = { name: nn, sourceField: field, fromEntities: [], distinctValues: 0 };
|
|
72312
|
+
dimByName.set(nn, dim);
|
|
72313
|
+
dimensions.push(dim);
|
|
72314
|
+
}
|
|
72315
|
+
if (!dim.fromEntities.includes(e6.name)) dim.fromEntities.push(e6.name);
|
|
72316
|
+
linkages.push({
|
|
72317
|
+
kind: "dimension",
|
|
72318
|
+
fromEntity: e6.name,
|
|
72319
|
+
fromField: field,
|
|
72320
|
+
toEntity: nn,
|
|
72321
|
+
toKey: "value",
|
|
72322
|
+
junction: `${e6.name}_${nn}`,
|
|
72323
|
+
matched: p3.distinct,
|
|
72324
|
+
unresolved: 0,
|
|
72325
|
+
confidence: 1
|
|
72326
|
+
});
|
|
72327
|
+
consume(e6.name, field);
|
|
72328
|
+
}
|
|
72329
|
+
}
|
|
72330
|
+
for (const dim of dimensions) {
|
|
72331
|
+
const all = /* @__PURE__ */ new Set();
|
|
72332
|
+
for (const name of dim.fromEntities) {
|
|
72333
|
+
const e6 = sources.find((s2) => s2.name === name);
|
|
72334
|
+
if (!e6) continue;
|
|
72335
|
+
for (const [f6, p3] of e6.profiles) {
|
|
72336
|
+
if (normalizeName(f6) === dim.name) for (const v2 of p3.valueSet) all.add(v2);
|
|
72337
|
+
}
|
|
72338
|
+
}
|
|
72339
|
+
dim.distinctValues = all.size;
|
|
72340
|
+
}
|
|
72341
|
+
const entities = sources.map((e6) => {
|
|
72342
|
+
const columns = [];
|
|
72343
|
+
for (const [field, p3] of e6.profiles) {
|
|
72344
|
+
if (p3.isArray) continue;
|
|
72345
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
72346
|
+
columns.push({ name: normalizeName(field), sourceKey: field, type: p3.type });
|
|
72347
|
+
}
|
|
72348
|
+
return {
|
|
72349
|
+
name: e6.name,
|
|
72350
|
+
sourceKey: e6.sourceKey,
|
|
72351
|
+
columns,
|
|
72352
|
+
naturalKey: e6.naturalKey ? normalizeName(e6.naturalKey) : null,
|
|
72353
|
+
naturalKeySource: e6.naturalKey,
|
|
72354
|
+
rowCount: e6.records.length,
|
|
72355
|
+
columnar: e6.columnar
|
|
72356
|
+
};
|
|
72357
|
+
});
|
|
72358
|
+
return { entities, dimensions, linkages, skipped };
|
|
72359
|
+
}
|
|
72360
|
+
|
|
72361
|
+
// src/import/dedupe-views.ts
|
|
72362
|
+
init_normalize();
|
|
72363
|
+
var SAMPLE2 = 300;
|
|
72364
|
+
var VIEW_MIN_OVERLAP = 0.8;
|
|
72365
|
+
function buildEntityData(plan, data) {
|
|
72366
|
+
return plan.entities.map((e6) => {
|
|
72367
|
+
const records = sourceRecords(data, e6);
|
|
72368
|
+
const colSet = /* @__PURE__ */ new Set();
|
|
72369
|
+
const colSource = /* @__PURE__ */ new Map();
|
|
72370
|
+
for (const r6 of records.slice(0, SAMPLE2)) {
|
|
72371
|
+
for (const k6 of Object.keys(r6)) {
|
|
72372
|
+
const n3 = normalizeName(k6);
|
|
72373
|
+
colSet.add(n3);
|
|
72374
|
+
if (!colSource.has(n3)) colSource.set(n3, k6);
|
|
72375
|
+
}
|
|
72376
|
+
}
|
|
72377
|
+
const normRows = records.map((r6) => {
|
|
72378
|
+
const o3 = {};
|
|
72379
|
+
for (const k6 of Object.keys(r6)) o3[normalizeName(k6)] = r6[k6];
|
|
72380
|
+
return o3;
|
|
72381
|
+
});
|
|
72382
|
+
return { name: e6.name, sourceKey: e6.sourceKey, cols: [...colSet], colSource, normRows };
|
|
72383
|
+
});
|
|
72384
|
+
}
|
|
72385
|
+
function pickIdentity(a6, shared) {
|
|
72386
|
+
let bestCol = null;
|
|
72387
|
+
let bestDistinct = -1;
|
|
72388
|
+
for (const c6 of shared) {
|
|
72389
|
+
const vals = /* @__PURE__ */ new Set();
|
|
72390
|
+
let textish = 0;
|
|
72391
|
+
let total = 0;
|
|
72392
|
+
for (const r6 of a6.normRows) {
|
|
72393
|
+
const v2 = r6[c6];
|
|
72394
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
72395
|
+
total++;
|
|
72396
|
+
if (typeof v2 === "string") textish++;
|
|
72397
|
+
vals.add(normalizeText(v2));
|
|
72398
|
+
}
|
|
72399
|
+
if (total === 0 || textish / total < 0.7) continue;
|
|
72400
|
+
if (vals.size > bestDistinct) {
|
|
72401
|
+
bestDistinct = vals.size;
|
|
72402
|
+
bestCol = c6;
|
|
72403
|
+
}
|
|
72404
|
+
}
|
|
72405
|
+
return bestCol;
|
|
72406
|
+
}
|
|
72407
|
+
function dedupeAndDetectViews(plan, data) {
|
|
72408
|
+
const entities = buildEntityData(plan, data);
|
|
72409
|
+
const views = [];
|
|
72410
|
+
const asView = /* @__PURE__ */ new Set();
|
|
72411
|
+
const colKeeps = [];
|
|
72412
|
+
for (const a6 of entities) {
|
|
72413
|
+
if (a6.cols.length < 2 || a6.normRows.length === 0) continue;
|
|
72414
|
+
const tabName = normalizeText(a6.sourceKey);
|
|
72415
|
+
if (!tabName) continue;
|
|
72416
|
+
const aColSet = new Set(a6.cols);
|
|
72417
|
+
let best = null;
|
|
72418
|
+
for (const b6 of entities) {
|
|
72419
|
+
if (b6.name === a6.name || asView.has(b6.name)) continue;
|
|
72420
|
+
if (b6.normRows.length < a6.normRows.length) continue;
|
|
72421
|
+
const bColSet = new Set(b6.cols);
|
|
72422
|
+
const shared = a6.cols.filter((c6) => bColSet.has(c6));
|
|
72423
|
+
if (shared.length < Math.max(2, Math.ceil(a6.cols.length * 0.5))) continue;
|
|
72424
|
+
const identity = pickIdentity(a6, shared);
|
|
72425
|
+
if (!identity) continue;
|
|
72426
|
+
const aIds = new Set(
|
|
72427
|
+
a6.normRows.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== "")
|
|
72428
|
+
);
|
|
72429
|
+
if (aIds.size === 0) continue;
|
|
72430
|
+
for (const disc of b6.cols) {
|
|
72431
|
+
if (aColSet.has(disc)) continue;
|
|
72432
|
+
const sub = b6.normRows.filter((r6) => normalizeText(r6[disc]) === tabName);
|
|
72433
|
+
if (sub.length === 0) continue;
|
|
72434
|
+
const bIds = new Set(sub.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== ""));
|
|
72435
|
+
let inter = 0;
|
|
72436
|
+
for (const id of aIds) if (bIds.has(id)) inter++;
|
|
72437
|
+
const overlap = inter / aIds.size;
|
|
72438
|
+
if (overlap < VIEW_MIN_OVERLAP) continue;
|
|
72439
|
+
const rawRow = sub.find((r6) => typeof r6[disc] === "string" || typeof r6[disc] === "number");
|
|
72440
|
+
const raw = rawRow ? rawRow[disc] : void 0;
|
|
72441
|
+
if (typeof raw !== "string" && typeof raw !== "number") continue;
|
|
72442
|
+
if (best === null || overlap > best.overlap || overlap === best.overlap && b6.cols.length > best.master.cols.length) {
|
|
72443
|
+
best = { master: b6, disc, value: String(raw), matched: sub.length, overlap };
|
|
72444
|
+
}
|
|
72445
|
+
}
|
|
72446
|
+
}
|
|
72447
|
+
if (!best) continue;
|
|
72448
|
+
views.push({
|
|
72449
|
+
name: a6.name,
|
|
72450
|
+
master: best.master.name,
|
|
72451
|
+
filterColumn: best.disc,
|
|
72452
|
+
filterValue: best.value,
|
|
72453
|
+
matchedRows: best.matched
|
|
72454
|
+
});
|
|
72455
|
+
asView.add(a6.name);
|
|
72456
|
+
colKeeps.push({ master: best.master, col: best.disc });
|
|
72457
|
+
}
|
|
72458
|
+
for (const { master, col } of colKeeps) {
|
|
72459
|
+
const masterEntity = plan.entities.find((e6) => e6.name === master.name);
|
|
72460
|
+
if (!masterEntity || masterEntity.columns.some((c6) => c6.name === col)) continue;
|
|
72461
|
+
masterEntity.columns.push({
|
|
72462
|
+
name: col,
|
|
72463
|
+
sourceKey: master.colSource.get(col) ?? col,
|
|
72464
|
+
type: inferFieldType(master.normRows.map((r6) => r6[col]))
|
|
72465
|
+
});
|
|
72466
|
+
}
|
|
72467
|
+
if (views.length === 0) return { plan, views };
|
|
72468
|
+
const nextPlan = {
|
|
72469
|
+
entities: plan.entities.filter((e6) => !asView.has(e6.name)),
|
|
72470
|
+
linkages: plan.linkages.filter((l4) => !asView.has(l4.fromEntity)),
|
|
72471
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.filter((n3) => !asView.has(n3)) })).filter((d6) => d6.fromEntities.length > 0),
|
|
72472
|
+
skipped: plan.skipped
|
|
72473
|
+
};
|
|
72474
|
+
return { plan: nextPlan, views };
|
|
72475
|
+
}
|
|
72476
|
+
|
|
72477
|
+
// src/import/excel.ts
|
|
72478
|
+
import { resolve as resolve10 } from "path";
|
|
72479
|
+
var HEADER_SCAN_ROWS = 25;
|
|
72480
|
+
function cellValue(v2) {
|
|
72481
|
+
if (v2 === null || v2 === void 0) return null;
|
|
72482
|
+
if (v2 instanceof Date) return v2.toISOString().slice(0, 10);
|
|
72483
|
+
if (typeof v2 === "object") {
|
|
72484
|
+
const o3 = v2;
|
|
72485
|
+
if ("result" in o3) return cellValue(o3.result);
|
|
72486
|
+
if ("text" in o3) return o3.text;
|
|
72487
|
+
if ("richText" in o3 && Array.isArray(o3.richText)) {
|
|
72488
|
+
return o3.richText.map((t8) => t8.text ?? "").join("");
|
|
72489
|
+
}
|
|
72490
|
+
return null;
|
|
72491
|
+
}
|
|
72492
|
+
return v2;
|
|
72493
|
+
}
|
|
72494
|
+
function isFilled(v2) {
|
|
72495
|
+
return v2 !== null && v2 !== void 0 && v2 !== "";
|
|
72496
|
+
}
|
|
72497
|
+
function sheetToRecords(ws) {
|
|
72498
|
+
const rowCount = ws.rowCount;
|
|
72499
|
+
const colCount = ws.columnCount;
|
|
72500
|
+
if (rowCount < 2 || colCount < 2) return [];
|
|
72501
|
+
const nonEmpty = (r6) => {
|
|
72502
|
+
let n3 = 0;
|
|
72503
|
+
for (let c6 = 1; c6 <= colCount; c6++) if (isFilled(cellValue(ws.getCell(r6, c6).value))) n3++;
|
|
72504
|
+
return n3;
|
|
72505
|
+
};
|
|
72506
|
+
const threshold = Math.max(3, Math.floor(colCount * 0.4));
|
|
72507
|
+
let headerRow = -1;
|
|
72508
|
+
for (let r6 = 1; r6 <= Math.min(HEADER_SCAN_ROWS, rowCount); r6++) {
|
|
72509
|
+
if (nonEmpty(r6) >= threshold && r6 < rowCount && nonEmpty(r6 + 1) >= 2) {
|
|
72510
|
+
headerRow = r6;
|
|
72511
|
+
break;
|
|
72512
|
+
}
|
|
72513
|
+
}
|
|
72514
|
+
if (headerRow < 0) return [];
|
|
72515
|
+
const cols = [];
|
|
72516
|
+
const seen = /* @__PURE__ */ new Set();
|
|
72517
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
72518
|
+
const hv = cellValue(ws.getCell(headerRow, c6).value);
|
|
72519
|
+
if (!isFilled(hv)) continue;
|
|
72520
|
+
const base = String(hv).replace(/\s+/g, " ").trim();
|
|
72521
|
+
if (!base) continue;
|
|
72522
|
+
let name = base;
|
|
72523
|
+
let i6 = 2;
|
|
72524
|
+
while (seen.has(name)) name = base + " " + String(i6++);
|
|
72525
|
+
seen.add(name);
|
|
72526
|
+
cols.push({ c: c6, name });
|
|
72527
|
+
}
|
|
72528
|
+
if (cols.length === 0) return [];
|
|
72529
|
+
const records = [];
|
|
72530
|
+
for (let r6 = headerRow + 1; r6 <= rowCount; r6++) {
|
|
72531
|
+
const row = {};
|
|
72532
|
+
let any = false;
|
|
72533
|
+
for (const { c: c6, name } of cols) {
|
|
72534
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
72535
|
+
if (isFilled(v2)) {
|
|
72536
|
+
row[name] = v2;
|
|
72537
|
+
any = true;
|
|
72538
|
+
}
|
|
72539
|
+
}
|
|
72540
|
+
if (!any) break;
|
|
72541
|
+
const first = cols[0] ? row[cols[0].name] : void 0;
|
|
72542
|
+
if (typeof first === "string" && /^total\b/i.test(first.trim())) continue;
|
|
72543
|
+
records.push(row);
|
|
72544
|
+
}
|
|
72545
|
+
return records;
|
|
72546
|
+
}
|
|
72547
|
+
var preambleCache = /* @__PURE__ */ new Map();
|
|
72548
|
+
function excelPreambleText(absPath) {
|
|
72549
|
+
return preambleCache.get(resolve10(absPath)) ?? "";
|
|
72550
|
+
}
|
|
72551
|
+
function sheetPreamble(ws) {
|
|
72552
|
+
const lines = [];
|
|
72553
|
+
const rowCount = Math.min(10, ws.rowCount);
|
|
72554
|
+
const colCount = Math.min(8, ws.columnCount);
|
|
72555
|
+
for (let r6 = 1; r6 <= rowCount; r6++) {
|
|
72556
|
+
const cells = [];
|
|
72557
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
72558
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
72559
|
+
if (isFilled(v2)) cells.push(String(v2));
|
|
72560
|
+
}
|
|
72561
|
+
if (cells.length) lines.push(cells.join(" "));
|
|
72562
|
+
}
|
|
72563
|
+
return lines.join("\n");
|
|
72564
|
+
}
|
|
72565
|
+
async function excelToRecords(absPath) {
|
|
72566
|
+
let mod;
|
|
72567
|
+
try {
|
|
72568
|
+
mod = await import("exceljs");
|
|
72569
|
+
} catch {
|
|
72570
|
+
throw new Error(
|
|
72571
|
+
'Reading Excel files needs the "exceljs" package \u2014 install it with: npm install exceljs'
|
|
72572
|
+
);
|
|
72573
|
+
}
|
|
72574
|
+
const ExcelJS = mod.default ?? mod;
|
|
72575
|
+
const wb = new ExcelJS.Workbook();
|
|
72576
|
+
await wb.xlsx.readFile(absPath);
|
|
72577
|
+
const out = {};
|
|
72578
|
+
const preamble = [];
|
|
72579
|
+
const props = wb.properties;
|
|
72580
|
+
if (props?.title) preamble.push(props.title);
|
|
72581
|
+
for (const ws of wb.worksheets) {
|
|
72582
|
+
preamble.push(ws.name, sheetPreamble(ws));
|
|
72583
|
+
const records = sheetToRecords(ws);
|
|
72584
|
+
if (records.length > 0) out[ws.name] = records;
|
|
72585
|
+
}
|
|
72586
|
+
preambleCache.set(resolve10(absPath), preamble.filter(Boolean).join("\n"));
|
|
72587
|
+
return out;
|
|
72588
|
+
}
|
|
72589
|
+
|
|
72590
|
+
// src/import/match.ts
|
|
72591
|
+
var BOOKKEEPING = /* @__PURE__ */ new Set(["id", "as_of", "content_key", "deleted_at"]);
|
|
72592
|
+
var MATCH_THRESHOLD = 0.6;
|
|
72593
|
+
function signature(columns) {
|
|
72594
|
+
const out = /* @__PURE__ */ new Set();
|
|
72595
|
+
for (const c6 of columns) {
|
|
72596
|
+
const n3 = normalizeName(c6);
|
|
72597
|
+
if (!n3 || BOOKKEEPING.has(n3) || n3.endsWith("_id")) continue;
|
|
72598
|
+
out.add(n3);
|
|
72599
|
+
}
|
|
72600
|
+
return out;
|
|
72601
|
+
}
|
|
72602
|
+
function containment(a6, b6) {
|
|
72603
|
+
if (a6.size === 0) return 0;
|
|
72604
|
+
let hit = 0;
|
|
72605
|
+
for (const c6 of a6) if (b6.has(c6)) hit++;
|
|
72606
|
+
return hit / a6.size;
|
|
72607
|
+
}
|
|
72608
|
+
function matchSchemaToExisting(existing, plan) {
|
|
72609
|
+
const ex = existing.map((t8) => ({ name: t8.name, sig: signature(t8.columns) }));
|
|
72610
|
+
const matches = [];
|
|
72611
|
+
const rename = {};
|
|
72612
|
+
for (const ent of plan.entities) {
|
|
72613
|
+
const sig = signature(ent.columns.map((c6) => c6.name));
|
|
72614
|
+
if (sig.size === 0) continue;
|
|
72615
|
+
let best = null;
|
|
72616
|
+
for (const t8 of ex) {
|
|
72617
|
+
if (normalizeName(t8.name) === normalizeName(ent.name)) {
|
|
72618
|
+
best = { name: t8.name, overlap: 1 };
|
|
72619
|
+
break;
|
|
72620
|
+
}
|
|
72621
|
+
const overlap = containment(sig, t8.sig);
|
|
72622
|
+
if (overlap > (best?.overlap ?? 0)) best = { name: t8.name, overlap };
|
|
72623
|
+
}
|
|
72624
|
+
if (best && best.overlap >= MATCH_THRESHOLD) {
|
|
72625
|
+
matches.push({ from: ent.name, to: best.name, overlap: best.overlap });
|
|
72626
|
+
if (best.name !== ent.name) rename[ent.name] = best.name;
|
|
72627
|
+
}
|
|
72628
|
+
}
|
|
72629
|
+
const totalEntities = plan.entities.length;
|
|
72630
|
+
const matchedCount = matches.length;
|
|
72631
|
+
const isKnownDocument = totalEntities > 0 && matchedCount >= Math.ceil(totalEntities / 2);
|
|
72632
|
+
return { matches, rename, matchedCount, totalEntities, isKnownDocument };
|
|
72633
|
+
}
|
|
72634
|
+
function renameEntities(plan, views, rename) {
|
|
72635
|
+
if (Object.keys(rename).length === 0) return { plan, views };
|
|
72636
|
+
const r6 = (n3) => rename[n3] ?? n3;
|
|
72637
|
+
return {
|
|
72638
|
+
plan: {
|
|
72639
|
+
...plan,
|
|
72640
|
+
entities: plan.entities.map((e6) => ({ ...e6, name: r6(e6.name) })),
|
|
72641
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.map(r6) })),
|
|
72642
|
+
linkages: plan.linkages.map((l4) => ({
|
|
72643
|
+
...l4,
|
|
72644
|
+
fromEntity: r6(l4.fromEntity),
|
|
72645
|
+
toEntity: r6(l4.toEntity),
|
|
72646
|
+
...l4.junction ? { junction: l4.junction } : {}
|
|
72647
|
+
}))
|
|
72648
|
+
},
|
|
72649
|
+
views: views.map((v2) => ({ ...v2, name: r6(v2.name), master: r6(v2.master) }))
|
|
72650
|
+
};
|
|
72651
|
+
}
|
|
72652
|
+
|
|
72653
|
+
// src/import/materialize.ts
|
|
72654
|
+
init_parser();
|
|
72655
|
+
import { createHash as createHash11 } from "crypto";
|
|
72656
|
+
import { existsSync as existsSync26 } from "fs";
|
|
72657
|
+
init_normalize();
|
|
72658
|
+
|
|
72659
|
+
// src/import/asof.ts
|
|
72660
|
+
var MONTHS2 = {
|
|
72661
|
+
jan: 1,
|
|
72662
|
+
january: 1,
|
|
72663
|
+
feb: 2,
|
|
72664
|
+
february: 2,
|
|
72665
|
+
mar: 3,
|
|
72666
|
+
march: 3,
|
|
72667
|
+
apr: 4,
|
|
72668
|
+
april: 4,
|
|
72669
|
+
may: 5,
|
|
72670
|
+
jun: 6,
|
|
72671
|
+
june: 6,
|
|
72672
|
+
jul: 7,
|
|
72673
|
+
july: 7,
|
|
72674
|
+
aug: 8,
|
|
72675
|
+
august: 8,
|
|
72676
|
+
sep: 9,
|
|
72677
|
+
sept: 9,
|
|
72678
|
+
september: 9,
|
|
72679
|
+
oct: 10,
|
|
72680
|
+
october: 10,
|
|
72681
|
+
nov: 11,
|
|
72682
|
+
november: 11,
|
|
72683
|
+
dec: 12,
|
|
72684
|
+
december: 12
|
|
72685
|
+
};
|
|
72686
|
+
var ASOF_KEYWORDS = /\b(as[ -]?of|as at|period (?:end(?:ed|ing)?|of)|fye|fiscal year end(?:ed|ing)?|year[ -]?end(?:ed|ing)?|quarter[ -]?end(?:ed|ing)?|valuation date|report(?:ing)? date|effective date|dated)\b/i;
|
|
72687
|
+
function isoFrom(y2, m4, d6) {
|
|
72688
|
+
if (m4 < 1 || m4 > 12 || d6 < 1 || d6 > 31) return null;
|
|
72689
|
+
if (y2 < 2010 || y2 > 2099) return null;
|
|
72690
|
+
return `${String(y2)}-${String(m4).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
72691
|
+
}
|
|
72692
|
+
function findDates(text) {
|
|
72693
|
+
const hits = [];
|
|
72694
|
+
const push = (date2, match, index) => {
|
|
72695
|
+
if (date2) hits.push({ date: date2, match, index });
|
|
72696
|
+
};
|
|
72697
|
+
for (const m4 of text.matchAll(/(20\d{2})[-._/](\d{1,2})[-._/](\d{1,2})/g)) {
|
|
72698
|
+
push(isoFrom(Number(m4[1]), Number(m4[2]), Number(m4[3])), m4[0], m4.index);
|
|
72699
|
+
}
|
|
72700
|
+
for (const m4 of text.matchAll(/(\d{1,2})[-._/](\d{1,2})[-._/](\d{2,4})/g)) {
|
|
72701
|
+
let y2 = Number(m4[3]);
|
|
72702
|
+
if (y2 < 100) y2 += 2e3;
|
|
72703
|
+
push(isoFrom(y2, Number(m4[1]), Number(m4[2])), m4[0], m4.index);
|
|
72704
|
+
}
|
|
72705
|
+
for (const m4 of text.matchAll(/([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(20\d{2})/g)) {
|
|
72706
|
+
const mon = MONTHS2[(m4[1] ?? "").toLowerCase()];
|
|
72707
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[2])), m4[0], m4.index);
|
|
72708
|
+
}
|
|
72709
|
+
for (const m4 of text.matchAll(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(20\d{2})/g)) {
|
|
72710
|
+
const mon = MONTHS2[(m4[2] ?? "").toLowerCase()];
|
|
72711
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[1])), m4[0], m4.index);
|
|
72712
|
+
}
|
|
72713
|
+
return hits;
|
|
72714
|
+
}
|
|
72715
|
+
function parseCellDate(value) {
|
|
72716
|
+
if (value instanceof Date) {
|
|
72717
|
+
return isoFrom(value.getUTCFullYear(), value.getUTCMonth() + 1, value.getUTCDate());
|
|
72718
|
+
}
|
|
72719
|
+
if (typeof value === "string") return findDates(value)[0]?.date ?? null;
|
|
72720
|
+
return null;
|
|
72721
|
+
}
|
|
72722
|
+
function scanText(text, label) {
|
|
72723
|
+
if (!text) return [];
|
|
72724
|
+
const out = [];
|
|
72725
|
+
for (const hit of findDates(text)) {
|
|
72726
|
+
const before = text.slice(Math.max(0, hit.index - 40), hit.index);
|
|
72727
|
+
const keyworded = ASOF_KEYWORDS.test(before) || ASOF_KEYWORDS.test(hit.match);
|
|
72728
|
+
const snippet = text.slice(Math.max(0, hit.index - 24), hit.index + hit.match.length + 4).replace(/\s+/g, " ").trim();
|
|
72729
|
+
out.push({
|
|
72730
|
+
date: hit.date,
|
|
72731
|
+
source: "content",
|
|
72732
|
+
confidence: keyworded ? 0.95 : 0.7,
|
|
72733
|
+
evidence: `${label}: "${snippet}"`
|
|
72734
|
+
});
|
|
72735
|
+
}
|
|
72736
|
+
return out;
|
|
72737
|
+
}
|
|
72738
|
+
function scanFilename(fileName) {
|
|
72739
|
+
if (!fileName) return [];
|
|
72740
|
+
const base = fileName.replace(/\.[A-Za-z0-9]+$/, "");
|
|
72741
|
+
return findDates(base).map((hit, i6, all) => ({
|
|
72742
|
+
date: hit.date,
|
|
72743
|
+
source: "filename",
|
|
72744
|
+
confidence: i6 === all.length - 1 ? 0.6 : 0.45,
|
|
72745
|
+
evidence: `file name: "${hit.match}"`
|
|
72746
|
+
}));
|
|
72747
|
+
}
|
|
72748
|
+
function detectAsOfCandidates(inputs) {
|
|
72749
|
+
const all = [];
|
|
72750
|
+
for (const t8 of inputs.texts ?? []) all.push(...scanText(t8.text, t8.label));
|
|
72751
|
+
if (inputs.fileName) all.push(...scanFilename(inputs.fileName));
|
|
72752
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
72753
|
+
for (const c6 of all) {
|
|
72754
|
+
const prev = byDate.get(c6.date);
|
|
72755
|
+
if (!prev || c6.confidence > prev.confidence) byDate.set(c6.date, c6);
|
|
72756
|
+
}
|
|
72757
|
+
return [...byDate.values()].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
72758
|
+
}
|
|
72759
|
+
|
|
72760
|
+
// src/import/materialize.ts
|
|
72761
|
+
function coerce2(v2, type) {
|
|
72762
|
+
if (v2 === null || v2 === void 0 || v2 === "") return null;
|
|
72763
|
+
if (type === "boolean") return v2 === true || v2 === "true" || v2 === 1 ? 1 : 0;
|
|
72764
|
+
return v2;
|
|
72765
|
+
}
|
|
72766
|
+
function contentKey(record) {
|
|
72767
|
+
const parts = Object.keys(record).sort().map((k6) => k6 + "=" + JSON.stringify(record[k6] ?? null));
|
|
72768
|
+
return createHash11("sha256").update(parts.join("|")).digest("hex");
|
|
72769
|
+
}
|
|
72770
|
+
function persistTable(configPath, name, fields) {
|
|
72771
|
+
if (!configPath || !existsSync26(configPath)) return;
|
|
72772
|
+
try {
|
|
72773
|
+
const doc = loadConfigDoc(configPath);
|
|
72774
|
+
doc.setIn(["entities", name], { fields, outputFile: name.toUpperCase() + ".md" });
|
|
72775
|
+
saveConfigDoc(configPath, doc);
|
|
72776
|
+
} catch {
|
|
72777
|
+
}
|
|
72778
|
+
}
|
|
72779
|
+
async function materializeImport(ctx, data, plan, views = [], opts = {}) {
|
|
72780
|
+
const { db, configPath } = ctx;
|
|
72781
|
+
const mode = opts.mode ?? "both";
|
|
72782
|
+
const doSchema = mode === "schema" || mode === "both";
|
|
72783
|
+
const doContents = mode === "contents" || mode === "both";
|
|
72784
|
+
const asOf = opts.asOf?.trim() ? opts.asOf.trim() : null;
|
|
72785
|
+
const asOfColumn = opts.asOfColumn?.trim() ? opts.asOfColumn.trim() : null;
|
|
72786
|
+
const dated = asOf !== null || asOfColumn !== null;
|
|
72787
|
+
const asOfSourceKey = (entity) => asOfColumn ? entity.columns.find((c6) => c6.name === asOfColumn)?.sourceKey ?? null : null;
|
|
72788
|
+
const rowAsOf = (entity, record) => {
|
|
72789
|
+
const sk = asOfSourceKey(entity);
|
|
72790
|
+
if (sk) {
|
|
72791
|
+
const d6 = parseCellDate(record[sk]);
|
|
72792
|
+
if (d6) return d6;
|
|
72793
|
+
}
|
|
72794
|
+
return asOf;
|
|
72795
|
+
};
|
|
72796
|
+
const recordKey = (entity, record) => {
|
|
72797
|
+
const a6 = rowAsOf(entity, record);
|
|
72798
|
+
return a6 ? contentKey({ ...record, __as_of: a6 }) : contentKey(record);
|
|
72799
|
+
};
|
|
72800
|
+
const scopedKey = (a6, keyVal) => (a6 ?? "") + "|" + normalizeText(keyVal);
|
|
72801
|
+
const report = async (p3) => {
|
|
72802
|
+
await opts.onProgress?.(p3);
|
|
72803
|
+
};
|
|
72804
|
+
const tablesCreated = [];
|
|
72805
|
+
const rowsByTable = {};
|
|
72806
|
+
const links = [];
|
|
72807
|
+
const viewResults = [];
|
|
72808
|
+
const byName = new Map(plan.entities.map((e6) => [e6.name, e6]));
|
|
72809
|
+
for (const entity of plan.entities) {
|
|
72810
|
+
const keyless = entity.naturalKey === null;
|
|
72811
|
+
const columns = { id: "TEXT PRIMARY KEY" };
|
|
72812
|
+
const fieldTypes = {};
|
|
72813
|
+
const cfgFields = { id: { type: "uuid", primaryKey: true } };
|
|
72814
|
+
for (const c6 of entity.columns) {
|
|
72815
|
+
columns[c6.name] = fieldToSqliteBaseType(c6.type);
|
|
72816
|
+
fieldTypes[c6.name] = c6.type;
|
|
72817
|
+
cfgFields[c6.name] = { type: c6.type };
|
|
72818
|
+
}
|
|
72819
|
+
const needsContentKey = keyless || dated;
|
|
72820
|
+
if (needsContentKey) {
|
|
72821
|
+
columns.content_key = "TEXT";
|
|
72822
|
+
cfgFields.content_key = { type: "text" };
|
|
72823
|
+
}
|
|
72824
|
+
if (dated) {
|
|
72825
|
+
columns.as_of = "TEXT";
|
|
72826
|
+
cfgFields.as_of = { type: "text" };
|
|
72827
|
+
}
|
|
72828
|
+
columns.deleted_at = "TEXT";
|
|
72829
|
+
cfgFields.deleted_at = { type: "text" };
|
|
72830
|
+
if (!db.getRegisteredTableNames().includes(entity.name)) tablesCreated.push(entity.name);
|
|
72831
|
+
await db.defineLate(entity.name, { columns, fieldTypes, primaryKey: "id" });
|
|
72832
|
+
persistTable(configPath, entity.name, cfgFields);
|
|
72833
|
+
await report({
|
|
72834
|
+
phase: "entities",
|
|
72835
|
+
table: entity.name,
|
|
72836
|
+
message: `Created table ${entity.name}`
|
|
72837
|
+
});
|
|
72838
|
+
if (doContents) {
|
|
72839
|
+
const records = sourceRecords(data, entity);
|
|
72840
|
+
const rows = records.map((r6) => {
|
|
72841
|
+
const row = {};
|
|
72842
|
+
for (const c6 of entity.columns) row[c6.name] = coerce2(r6[c6.sourceKey], c6.type);
|
|
72843
|
+
if (needsContentKey) row.content_key = recordKey(entity, r6);
|
|
72844
|
+
if (dated) row.as_of = rowAsOf(entity, r6);
|
|
72845
|
+
return row;
|
|
72846
|
+
});
|
|
72847
|
+
await db.seed({
|
|
72848
|
+
data: rows,
|
|
72849
|
+
table: entity.name,
|
|
72850
|
+
naturalKey: dated ? "content_key" : entity.naturalKey ?? "content_key"
|
|
72851
|
+
});
|
|
72852
|
+
const n3 = await db.count(entity.name);
|
|
72853
|
+
rowsByTable[entity.name] = n3;
|
|
72854
|
+
await report({
|
|
72855
|
+
phase: "entities",
|
|
72856
|
+
table: entity.name,
|
|
72857
|
+
count: n3,
|
|
72858
|
+
message: `Loaded ${String(n3)} rows into ${entity.name}`
|
|
72859
|
+
});
|
|
72860
|
+
}
|
|
72861
|
+
}
|
|
72862
|
+
for (const dim of plan.dimensions) {
|
|
72863
|
+
if (!db.getRegisteredTableNames().includes(dim.name)) tablesCreated.push(dim.name);
|
|
72864
|
+
await db.defineLate(dim.name, {
|
|
72865
|
+
columns: { id: "TEXT PRIMARY KEY", value: "TEXT", deleted_at: "TEXT" },
|
|
72866
|
+
fieldTypes: { value: "text" },
|
|
72867
|
+
primaryKey: "id"
|
|
72868
|
+
});
|
|
72869
|
+
persistTable(configPath, dim.name, {
|
|
72870
|
+
id: { type: "uuid", primaryKey: true },
|
|
72871
|
+
value: { type: "text" },
|
|
72872
|
+
deleted_at: { type: "text" }
|
|
72873
|
+
});
|
|
72874
|
+
if (doSchema) {
|
|
72875
|
+
const values = /* @__PURE__ */ new Map();
|
|
72876
|
+
for (const ename of dim.fromEntities) {
|
|
72877
|
+
const ent = byName.get(ename);
|
|
72878
|
+
if (!ent) continue;
|
|
72879
|
+
const records = sourceRecords(data, ent);
|
|
72880
|
+
const first = records[0];
|
|
72881
|
+
const srcKey = first ? Object.keys(first).find((k6) => normalizeName(k6) === dim.name) : void 0;
|
|
72882
|
+
if (!srcKey) continue;
|
|
72883
|
+
for (const r6 of records) {
|
|
72884
|
+
const v2 = r6[srcKey];
|
|
72885
|
+
if (typeof v2 !== "string" && typeof v2 !== "number") continue;
|
|
72886
|
+
const key = normalizeText(v2);
|
|
72887
|
+
if (key !== "" && !values.has(key)) values.set(key, String(v2));
|
|
72888
|
+
}
|
|
72889
|
+
}
|
|
72890
|
+
await db.seed({
|
|
72891
|
+
data: [...values.values()].map((value) => ({ value })),
|
|
72892
|
+
table: dim.name,
|
|
72893
|
+
naturalKey: "value"
|
|
72894
|
+
});
|
|
72895
|
+
const n3 = await db.count(dim.name);
|
|
72896
|
+
rowsByTable[dim.name] = n3;
|
|
72897
|
+
await report({
|
|
72898
|
+
phase: "dimensions",
|
|
72899
|
+
table: dim.name,
|
|
72900
|
+
count: n3,
|
|
72901
|
+
message: `Dimension ${dim.name}: ${String(n3)} values`
|
|
72902
|
+
});
|
|
72903
|
+
}
|
|
72904
|
+
}
|
|
72905
|
+
const idMapCache = /* @__PURE__ */ new Map();
|
|
72906
|
+
async function idMap(table, keyCol, datedTarget) {
|
|
72907
|
+
const cacheKey = table + ":" + keyCol + ":" + (datedTarget ? "D" : "");
|
|
72908
|
+
const cached = idMapCache.get(cacheKey);
|
|
72909
|
+
if (cached) return cached;
|
|
72910
|
+
const map = /* @__PURE__ */ new Map();
|
|
72911
|
+
for (const r6 of await db.query(table)) {
|
|
72912
|
+
const k6 = r6[keyCol];
|
|
72913
|
+
if (k6 === null || k6 === void 0) continue;
|
|
72914
|
+
const mapKey = datedTarget ? scopedKey(r6.as_of, k6) : normalizeText(k6);
|
|
72915
|
+
map.set(mapKey, String(r6.id));
|
|
72916
|
+
}
|
|
72917
|
+
idMapCache.set(cacheKey, map);
|
|
72918
|
+
return map;
|
|
72919
|
+
}
|
|
72920
|
+
for (const link of plan.linkages) {
|
|
72921
|
+
const from = byName.get(link.fromEntity);
|
|
72922
|
+
if (!from) continue;
|
|
72923
|
+
const jName = link.junction ?? `${link.fromEntity}_${link.toEntity}`;
|
|
72924
|
+
const fromFk = `${link.fromEntity}_id`;
|
|
72925
|
+
const toFk = `${link.toEntity}_id`;
|
|
72926
|
+
const jCols = {
|
|
72927
|
+
id: "TEXT PRIMARY KEY",
|
|
72928
|
+
[fromFk]: "TEXT",
|
|
72929
|
+
[toFk]: "TEXT"
|
|
72930
|
+
};
|
|
72931
|
+
const jCfg = {
|
|
72932
|
+
id: { type: "uuid", primaryKey: true },
|
|
72933
|
+
[fromFk]: { type: "uuid", ref: link.fromEntity },
|
|
72934
|
+
[toFk]: { type: "uuid", ref: link.toEntity }
|
|
72935
|
+
};
|
|
72936
|
+
if (dated) {
|
|
72937
|
+
jCols.as_of = "TEXT";
|
|
72938
|
+
jCfg.as_of = { type: "text" };
|
|
72939
|
+
}
|
|
72940
|
+
if (!db.getRegisteredTableNames().includes(jName)) tablesCreated.push(jName);
|
|
72941
|
+
await db.defineLate(jName, { columns: jCols, primaryKey: "id" });
|
|
72942
|
+
persistTable(configPath, jName, jCfg);
|
|
72943
|
+
if (!doContents) continue;
|
|
72944
|
+
const fromKeyCol = from.naturalKey ?? "content_key";
|
|
72945
|
+
const toIsEntity = byName.has(link.toEntity);
|
|
72946
|
+
const fromMap = await idMap(link.fromEntity, fromKeyCol, dated);
|
|
72947
|
+
const toMap = await idMap(link.toEntity, link.toKey, toIsEntity && dated);
|
|
72948
|
+
const seen = /* @__PURE__ */ new Set();
|
|
72949
|
+
for (const r6 of await db.query(jName)) {
|
|
72950
|
+
seen.add(String(r6[fromFk]) + "|" + String(r6[toFk]));
|
|
72951
|
+
}
|
|
72952
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
72953
|
+
let created = 0;
|
|
72954
|
+
for (const record of sourceRecords(data, from)) {
|
|
72955
|
+
const a6 = rowAsOf(from, record);
|
|
72956
|
+
const fromKeyVal = from.naturalKey === null ? recordKey(from, record) : record[from.naturalKeySource ?? ""];
|
|
72957
|
+
const fromId = fromMap.get(dated ? scopedKey(a6, fromKeyVal) : normalizeText(fromKeyVal));
|
|
72958
|
+
if (!fromId) continue;
|
|
72959
|
+
const raw = record[link.fromField];
|
|
72960
|
+
const refs = Array.isArray(raw) ? raw : [raw];
|
|
72961
|
+
for (const ref of refs) {
|
|
72962
|
+
if (ref === null || ref === void 0 || ref === "") continue;
|
|
72963
|
+
const toId = toMap.get(toIsEntity && dated ? scopedKey(a6, ref) : normalizeText(ref));
|
|
72964
|
+
if (!toId) {
|
|
72965
|
+
unresolved.add(normalizeText(ref));
|
|
72966
|
+
continue;
|
|
72967
|
+
}
|
|
72968
|
+
const edge = fromId + "|" + toId;
|
|
72969
|
+
if (seen.has(edge)) continue;
|
|
72970
|
+
seen.add(edge);
|
|
72971
|
+
await db.insert(
|
|
72972
|
+
jName,
|
|
72973
|
+
dated ? { [fromFk]: fromId, [toFk]: toId, as_of: a6 } : { [fromFk]: fromId, [toFk]: toId }
|
|
72974
|
+
);
|
|
72975
|
+
created++;
|
|
72976
|
+
}
|
|
72977
|
+
}
|
|
72978
|
+
rowsByTable[jName] = created;
|
|
72979
|
+
links.push({ junction: jName, created, unresolved: unresolved.size });
|
|
72980
|
+
await report({
|
|
72981
|
+
phase: "links",
|
|
72982
|
+
table: jName,
|
|
72983
|
+
count: created,
|
|
72984
|
+
message: `Linked ${String(created)} ${jName}`
|
|
72985
|
+
});
|
|
72986
|
+
}
|
|
72987
|
+
if (doSchema) {
|
|
72988
|
+
for (const v2 of views) {
|
|
72989
|
+
const filt = v2.filterValue.replace(/'/g, "''");
|
|
72990
|
+
await execSql(db, `DROP VIEW IF EXISTS "${v2.name}"`);
|
|
72991
|
+
await execSql(
|
|
72992
|
+
db,
|
|
72993
|
+
`CREATE VIEW "${v2.name}" AS SELECT * FROM "${v2.master}" WHERE "${v2.filterColumn}" = '${filt}'`
|
|
72994
|
+
);
|
|
72995
|
+
const cols = await db.introspectColumns(v2.name);
|
|
72996
|
+
await db.defineLate(v2.name, {
|
|
72997
|
+
columns: Object.fromEntries(cols.map((c6) => [c6, "TEXT"])),
|
|
72998
|
+
render: () => ""
|
|
72999
|
+
});
|
|
73000
|
+
if (!tablesCreated.includes(v2.name)) tablesCreated.push(v2.name);
|
|
73001
|
+
const rows = await db.count(v2.name);
|
|
73002
|
+
rowsByTable[v2.name] = rows;
|
|
73003
|
+
viewResults.push({ name: v2.name, master: v2.master, rows });
|
|
73004
|
+
await report({
|
|
73005
|
+
phase: "views",
|
|
73006
|
+
table: v2.name,
|
|
73007
|
+
count: rows,
|
|
73008
|
+
message: `View ${v2.name}: ${String(rows)} rows`
|
|
73009
|
+
});
|
|
73010
|
+
}
|
|
73011
|
+
}
|
|
73012
|
+
await report({ phase: "done", message: "Import complete" });
|
|
73013
|
+
return { mode, asOf, asOfColumn, tablesCreated, rowsByTable, links, views: viewResults };
|
|
73014
|
+
}
|
|
73015
|
+
|
|
73016
|
+
// src/gui/import-auto.ts
|
|
73017
|
+
init_native_entities();
|
|
73018
|
+
|
|
73019
|
+
// src/gui/import-detect.ts
|
|
73020
|
+
import { basename as basename10 } from "path";
|
|
73021
|
+
|
|
73022
|
+
// src/gui/ai/asof-llm.ts
|
|
73023
|
+
init_assistant_routes();
|
|
73024
|
+
init_chat();
|
|
73025
|
+
var MAX_CHARS = 6e3;
|
|
73026
|
+
var SYSTEM = 'You extract the single "as of" / report / snapshot / period-end date from the text of a data file (a financial statement, track record, export, etc.). Reply with ONLY that date as ISO YYYY-MM-DD, or the exact word NONE if the text has no such date. Output nothing else \u2014 no prose, no quotes.';
|
|
73027
|
+
function parseLlmDate(reply) {
|
|
73028
|
+
if (!reply) return null;
|
|
73029
|
+
const m4 = /(20\d{2})-(\d{2})-(\d{2})/.exec(reply);
|
|
73030
|
+
if (!m4) return null;
|
|
73031
|
+
const y2 = Number(m4[1]);
|
|
73032
|
+
const mo = Number(m4[2]);
|
|
73033
|
+
const d6 = Number(m4[3]);
|
|
73034
|
+
if (mo < 1 || mo > 12 || d6 < 1 || d6 > 31 || y2 < 2010 || y2 > 2099) return null;
|
|
73035
|
+
return `${String(y2)}-${String(mo).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
73036
|
+
}
|
|
73037
|
+
async function asOfFromLlm(db, text) {
|
|
73038
|
+
const trimmed = text.trim();
|
|
73039
|
+
if (!trimmed) return null;
|
|
73040
|
+
try {
|
|
73041
|
+
const auth = await resolveClaudeAuth(db);
|
|
73042
|
+
if (!auth) return null;
|
|
73043
|
+
const client = createAnthropicClient(auth);
|
|
73044
|
+
const result = await client.runTurn({
|
|
73045
|
+
model: DEFAULT_MODEL2,
|
|
73046
|
+
system: SYSTEM,
|
|
73047
|
+
temperature: 0,
|
|
73048
|
+
tools: [],
|
|
73049
|
+
messages: [{ role: "user", content: `File text:
|
|
73050
|
+
${trimmed.slice(0, MAX_CHARS)}` }],
|
|
73051
|
+
onText: () => {
|
|
73052
|
+
}
|
|
73053
|
+
});
|
|
73054
|
+
const date2 = parseLlmDate(result.text);
|
|
73055
|
+
return date2 ? { date: date2, source: "llm", confidence: 0.85, evidence: "Claude read the file" } : null;
|
|
73056
|
+
} catch (e6) {
|
|
73057
|
+
console.warn("[import] as-of LLM fallback failed:", e6.message);
|
|
73058
|
+
return null;
|
|
73059
|
+
}
|
|
73060
|
+
}
|
|
73061
|
+
|
|
73062
|
+
// src/gui/import-detect.ts
|
|
73063
|
+
async function detectImportAsOf(db, data, opts = {}) {
|
|
73064
|
+
const fileName = opts.fileName ?? (opts.abs ? basename10(opts.abs).replace(/^[0-9a-f]{8}-/, "") : "");
|
|
73065
|
+
const texts = [];
|
|
73066
|
+
for (const [k6, v2] of Object.entries(data)) {
|
|
73067
|
+
if (!Array.isArray(v2)) texts.push({ label: "data", text: `${k6}: ${JSON.stringify(v2)}` });
|
|
73068
|
+
}
|
|
73069
|
+
if (opts.abs && /\.xlsx?$/i.test(opts.abs)) {
|
|
73070
|
+
const pre = excelPreambleText(opts.abs);
|
|
73071
|
+
if (pre) texts.push({ label: "title", text: pre });
|
|
73072
|
+
}
|
|
73073
|
+
let candidates = detectAsOfCandidates({ fileName, texts });
|
|
73074
|
+
if (!candidates[0] || candidates[0].confidence < 0.7) {
|
|
73075
|
+
const llm = await asOfFromLlm(db, texts.map((t8) => t8.text).join("\n"));
|
|
73076
|
+
if (llm) candidates = [...candidates, llm].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
73077
|
+
}
|
|
73078
|
+
return candidates;
|
|
73079
|
+
}
|
|
73080
|
+
|
|
73081
|
+
// src/import/asof-columns.ts
|
|
73082
|
+
var STRONG_NAME = /(as[_ -]?of|as[_ -]?at|report(?:ing)?[_ -]?date|valuation[_ -]?date|effective[_ -]?date|period[_ -]?end|snapshot[_ -]?date|statement[_ -]?date|fye)/i;
|
|
73083
|
+
var WEAK_NAME = /(^|_)(date|period|quarter|asof)($|_)/i;
|
|
73084
|
+
function detectAsOfColumns(data, plan) {
|
|
73085
|
+
const out = [];
|
|
73086
|
+
for (const entity of plan.entities) {
|
|
73087
|
+
const records = sourceRecords(data, entity);
|
|
73088
|
+
if (records.length < 2) continue;
|
|
73089
|
+
for (const col of entity.columns) {
|
|
73090
|
+
const strong = STRONG_NAME.test(col.name);
|
|
73091
|
+
const weak = WEAK_NAME.test(col.name);
|
|
73092
|
+
if (!strong && !weak) continue;
|
|
73093
|
+
const vals = records.map((r6) => r6[col.sourceKey]).filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
73094
|
+
if (vals.length < Math.max(3, Math.floor(records.length * 0.5))) continue;
|
|
73095
|
+
const dates = vals.map(parseCellDate).filter((d6) => d6 !== null);
|
|
73096
|
+
if (dates.length / vals.length < 0.8) continue;
|
|
73097
|
+
const distinctDates = new Set(dates).size;
|
|
73098
|
+
const typed = col.type === "date" || col.type === "datetime";
|
|
73099
|
+
let confidence = strong ? 0.9 : 0.6;
|
|
73100
|
+
if (typed) confidence += 0.03;
|
|
73101
|
+
if (distinctDates > 1) confidence += 0.04;
|
|
73102
|
+
out.push({
|
|
73103
|
+
entity: entity.name,
|
|
73104
|
+
column: col.name,
|
|
73105
|
+
confidence: Math.min(confidence, 0.97),
|
|
73106
|
+
distinctDates,
|
|
73107
|
+
evidence: `column "${col.name}" \u2014 ${String(distinctDates)} distinct date${distinctDates === 1 ? "" : "s"} across ${String(vals.length)} rows`
|
|
73108
|
+
});
|
|
73109
|
+
}
|
|
73110
|
+
}
|
|
73111
|
+
return out.sort((a6, b6) => b6.confidence - a6.confidence);
|
|
73112
|
+
}
|
|
73113
|
+
|
|
73114
|
+
// src/gui/import-auto.ts
|
|
73115
|
+
function existingDataTables(db) {
|
|
73116
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
73117
|
+
const out = [];
|
|
73118
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
73119
|
+
if (native.has(t8)) continue;
|
|
73120
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
73121
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
73122
|
+
}
|
|
73123
|
+
return out;
|
|
73124
|
+
}
|
|
73125
|
+
async function readStructured(abs, name) {
|
|
73126
|
+
if (/\.xlsx?$/i.test(name)) return excelToRecords(abs);
|
|
73127
|
+
return JSON.parse(readFileSync22(abs, "utf8"));
|
|
73128
|
+
}
|
|
73129
|
+
async function autoImportStructured(db, configPath, abs, name) {
|
|
73130
|
+
if (!/\.(xlsx?|json)$/i.test(name)) return null;
|
|
73131
|
+
let data;
|
|
73132
|
+
try {
|
|
73133
|
+
data = await readStructured(abs, name);
|
|
73134
|
+
} catch {
|
|
73135
|
+
return null;
|
|
73136
|
+
}
|
|
73137
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
73138
|
+
inferSchema(data),
|
|
73139
|
+
data
|
|
73140
|
+
);
|
|
73141
|
+
if (inferredPlan.entities.length === 0) return null;
|
|
73142
|
+
const schemaMatch = matchSchemaToExisting(existingDataTables(db), inferredPlan);
|
|
73143
|
+
const asOfCandidates = await detectImportAsOf(db, data, { abs, fileName: name });
|
|
73144
|
+
const asOf = asOfCandidates[0]?.date ?? null;
|
|
73145
|
+
const asOfColumns = detectAsOfColumns(data, inferredPlan);
|
|
73146
|
+
const proposal = {
|
|
73147
|
+
plan: inferredPlan,
|
|
73148
|
+
views: inferredViews,
|
|
73149
|
+
asOfCandidates,
|
|
73150
|
+
asOfColumns,
|
|
73151
|
+
schemaMatch,
|
|
73152
|
+
matchedCount: schemaMatch.matchedCount,
|
|
73153
|
+
totalEntities: schemaMatch.totalEntities,
|
|
73154
|
+
tables: [],
|
|
73155
|
+
rows: 0
|
|
73156
|
+
};
|
|
73157
|
+
if (!schemaMatch.isKnownDocument) {
|
|
73158
|
+
return { imported: false, reason: "new-dataset", asOf, ...proposal };
|
|
73159
|
+
}
|
|
73160
|
+
if (!asOf) {
|
|
73161
|
+
return { imported: false, reason: "needs-confirm", asOf: null, ...proposal };
|
|
73162
|
+
}
|
|
73163
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, schemaMatch.rename);
|
|
73164
|
+
const result = await materializeImport({ db, configPath }, data, plan, views, { asOf });
|
|
73165
|
+
const rows = Object.values(result.rowsByTable).reduce((a6, b6) => a6 + b6, 0);
|
|
73166
|
+
return {
|
|
73167
|
+
imported: true,
|
|
73168
|
+
asOf,
|
|
73169
|
+
matchedCount: schemaMatch.matchedCount,
|
|
73170
|
+
totalEntities: schemaMatch.totalEntities,
|
|
73171
|
+
tables: Object.keys(result.rowsByTable),
|
|
73172
|
+
rows
|
|
73173
|
+
};
|
|
73174
|
+
}
|
|
73175
|
+
|
|
73176
|
+
// src/gui/ingest-routes.ts
|
|
71598
73177
|
var MIME_BY_EXT = {
|
|
71599
73178
|
".pdf": "application/pdf",
|
|
71600
73179
|
".png": "image/png",
|
|
@@ -71730,7 +73309,6 @@ function looksLikeUrl(s2) {
|
|
|
71730
73309
|
const t8 = s2.trim();
|
|
71731
73310
|
return /^https?:\/\/\S+$/i.test(t8) && !/\s/.test(t8);
|
|
71732
73311
|
}
|
|
71733
|
-
var MAX_INGEST_BYTES = 5e7;
|
|
71734
73312
|
function readBuffer2(req, maxBytes = MAX_INGEST_BYTES) {
|
|
71735
73313
|
return new Promise((resolve_, reject) => {
|
|
71736
73314
|
const chunks = [];
|
|
@@ -71792,9 +73370,15 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71792
73370
|
const tmp = join28(tmpdir2(), `lattice-ingest-${crypto.randomUUID()}${extname2(name2)}`);
|
|
71793
73371
|
let result;
|
|
71794
73372
|
let blob = null;
|
|
73373
|
+
let autoImport = null;
|
|
71795
73374
|
try {
|
|
71796
73375
|
await writeFile2(tmp, buf);
|
|
71797
73376
|
result = await extractSource(ctx.db, tmp, mime2, name2);
|
|
73377
|
+
try {
|
|
73378
|
+
autoImport = await autoImportStructured(ctx.db, ctx.configPath ?? null, tmp, name2);
|
|
73379
|
+
} catch (e6) {
|
|
73380
|
+
console.warn("[ingest] auto-import skipped:", e6.message);
|
|
73381
|
+
}
|
|
71798
73382
|
if (ctx.latticeRoot && !realPath && shouldRetainUploadBlob(mime2, name2)) {
|
|
71799
73383
|
try {
|
|
71800
73384
|
const meta = await attachBlob(tmp, ctx.latticeRoot);
|
|
@@ -71810,7 +73394,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71810
73394
|
let s3Status = null;
|
|
71811
73395
|
const s3cfg = resolveActiveS3Config(ctx.configPath);
|
|
71812
73396
|
if (s3cfg) {
|
|
71813
|
-
const sha256 = blob?.sha256 ??
|
|
73397
|
+
const sha256 = blob?.sha256 ?? createHash12("sha256").update(buf).digest("hex");
|
|
71814
73398
|
const key = s3Key(s3cfg.prefix, sha256);
|
|
71815
73399
|
try {
|
|
71816
73400
|
const store = await createS3Store(s3cfg);
|
|
@@ -71833,7 +73417,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71833
73417
|
}
|
|
71834
73418
|
}
|
|
71835
73419
|
const fileId = crypto.randomUUID();
|
|
71836
|
-
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ??
|
|
73420
|
+
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ?? createHash12("sha256").update(buf).digest("hex");
|
|
71837
73421
|
const uploadRow = {
|
|
71838
73422
|
id: fileId,
|
|
71839
73423
|
...fileIdentity(name2, fileId),
|
|
@@ -71871,6 +73455,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71871
73455
|
},
|
|
71872
73456
|
forcePrivate2 ? "private" : void 0
|
|
71873
73457
|
);
|
|
73458
|
+
if (autoImport?.reason) autoImport.fileId = id2;
|
|
71874
73459
|
try {
|
|
71875
73460
|
const dedupCtx = {
|
|
71876
73461
|
db: ctx.db,
|
|
@@ -71898,6 +73483,15 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71898
73483
|
e6 instanceof Error ? e6.message : String(e6)
|
|
71899
73484
|
);
|
|
71900
73485
|
}
|
|
73486
|
+
if (autoImport?.imported) {
|
|
73487
|
+
ctx.feed.publish({
|
|
73488
|
+
table: autoImport.tables[0] ?? "files",
|
|
73489
|
+
op: "insert",
|
|
73490
|
+
rowId: null,
|
|
73491
|
+
source: "system",
|
|
73492
|
+
summary: `Imported the ${autoImport.asOf ?? ""} snapshot of "${name2}" \u2014 ${String(autoImport.rows)} rows across ${String(autoImport.tables.length)} tables`
|
|
73493
|
+
});
|
|
73494
|
+
}
|
|
71901
73495
|
let suggestedLinks = [];
|
|
71902
73496
|
if (!result.skip) {
|
|
71903
73497
|
const links = await enrichOrFail(mctx, ctx.db, id2, result.text, name2, ctx, res, forcePrivate2);
|
|
@@ -71910,6 +73504,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71910
73504
|
id: id2,
|
|
71911
73505
|
extraction_status: result.skip ? "skipped" : "extracted",
|
|
71912
73506
|
suggestedLinks,
|
|
73507
|
+
...autoImport ? { autoImport } : {},
|
|
71913
73508
|
// Present only when S3 is enabled for this workspace. 'failed' tells the
|
|
71914
73509
|
// uploader the bytes did NOT reach the shared bucket — other members would
|
|
71915
73510
|
// 404 until it's re-uploaded — so the GUI can warn rather than imply a
|
|
@@ -71995,7 +73590,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71995
73590
|
sendJson(res, { error: "path is required" }, 400);
|
|
71996
73591
|
return true;
|
|
71997
73592
|
}
|
|
71998
|
-
const abs =
|
|
73593
|
+
const abs = resolve11(rawPath);
|
|
71999
73594
|
let size = 0;
|
|
72000
73595
|
try {
|
|
72001
73596
|
const st = statSync8(abs);
|
|
@@ -72012,7 +73607,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
72012
73607
|
sendJson(res, { error: "file too large" }, 413);
|
|
72013
73608
|
return true;
|
|
72014
73609
|
}
|
|
72015
|
-
const name =
|
|
73610
|
+
const name = basename11(abs);
|
|
72016
73611
|
const mime = mimeFor(name);
|
|
72017
73612
|
const localFileId = crypto.randomUUID();
|
|
72018
73613
|
const localRow = {
|
|
@@ -72078,6 +73673,146 @@ ${err.stack ?? ""}`
|
|
|
72078
73673
|
return true;
|
|
72079
73674
|
}
|
|
72080
73675
|
|
|
73676
|
+
// src/gui/import-routes.ts
|
|
73677
|
+
init_adapter();
|
|
73678
|
+
init_http();
|
|
73679
|
+
import { existsSync as existsSync27, readFileSync as readFileSync23, statSync as statSync9 } from "fs";
|
|
73680
|
+
import { isAbsolute as isAbsolute4, join as join29 } from "path";
|
|
73681
|
+
init_native_entities();
|
|
73682
|
+
function badRequest(message) {
|
|
73683
|
+
const e6 = new Error(message);
|
|
73684
|
+
e6.statusCode = 400;
|
|
73685
|
+
return e6;
|
|
73686
|
+
}
|
|
73687
|
+
function localPathOf2(row, latticeRoot) {
|
|
73688
|
+
if (row.ref_kind === "local_ref" && row.ref_uri) return row.ref_uri;
|
|
73689
|
+
if ((row.ref_kind === "blob" || row.ref_kind === "cloud_ref") && row.blob_path) {
|
|
73690
|
+
return isAbsolute4(row.blob_path) ? row.blob_path : latticeRoot ? join29(latticeRoot, row.blob_path) : null;
|
|
73691
|
+
}
|
|
73692
|
+
return null;
|
|
73693
|
+
}
|
|
73694
|
+
function existingDataTables2(db) {
|
|
73695
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
73696
|
+
const out = [];
|
|
73697
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
73698
|
+
if (native.has(t8)) continue;
|
|
73699
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
73700
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
73701
|
+
}
|
|
73702
|
+
return out;
|
|
73703
|
+
}
|
|
73704
|
+
async function readImportSourceFromFile(db, fileId, latticeRoot) {
|
|
73705
|
+
const row = await getAsyncOrSync(
|
|
73706
|
+
db.adapter,
|
|
73707
|
+
`SELECT "id","original_name","mime","ref_kind","ref_uri","blob_path"
|
|
73708
|
+
FROM "files" WHERE "id" = ? AND "deleted_at" IS NULL LIMIT 1`,
|
|
73709
|
+
[fileId]
|
|
73710
|
+
);
|
|
73711
|
+
if (!row) throw badRequest("Unknown import file: " + fileId);
|
|
73712
|
+
const path2 = localPathOf2(row, latticeRoot);
|
|
73713
|
+
if (!path2 || !existsSync27(path2)) {
|
|
73714
|
+
throw badRequest("The import file\u2019s bytes are not available locally.");
|
|
73715
|
+
}
|
|
73716
|
+
const sizeBytes = statSync9(path2).size;
|
|
73717
|
+
if (sizeBytes > MAX_INGEST_BYTES) {
|
|
73718
|
+
throw badRequest(
|
|
73719
|
+
`The import file is too large (${String(Math.round(sizeBytes / 1e6))} MB); the limit is ${String(Math.round(MAX_INGEST_BYTES / 1e6))} MB.`
|
|
73720
|
+
);
|
|
73721
|
+
}
|
|
73722
|
+
const name = row.original_name ?? "";
|
|
73723
|
+
const mime = row.mime ?? "";
|
|
73724
|
+
if (/\.xlsx?$/i.test(name) || mime.includes("spreadsheet") || mime.includes("excel")) {
|
|
73725
|
+
return excelToRecords(path2);
|
|
73726
|
+
}
|
|
73727
|
+
let parsed;
|
|
73728
|
+
try {
|
|
73729
|
+
parsed = JSON.parse(readFileSync23(path2, "utf8"));
|
|
73730
|
+
} catch {
|
|
73731
|
+
throw badRequest("The import file is not valid JSON.");
|
|
73732
|
+
}
|
|
73733
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
73734
|
+
throw badRequest("Expected a JSON object whose keys are record arrays.");
|
|
73735
|
+
}
|
|
73736
|
+
return parsed;
|
|
73737
|
+
}
|
|
73738
|
+
async function dispatchImportRoute(req, res, deps) {
|
|
73739
|
+
const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
|
|
73740
|
+
if (req.method !== "POST" || pathname !== "/api/import/apply") return false;
|
|
73741
|
+
const body = await readJson(req).catch(() => ({}));
|
|
73742
|
+
const fileId = typeof body.fileId === "string" ? body.fileId : "";
|
|
73743
|
+
const mode = body.mode === "schema" || body.mode === "contents" ? body.mode : "both";
|
|
73744
|
+
const asOf = typeof body.asOf === "string" && /^\d{4}-\d{2}-\d{2}$/.test(body.asOf.trim()) ? body.asOf.trim() : null;
|
|
73745
|
+
const asOfColumn = typeof body.asOfColumn === "string" && body.asOfColumn.trim() ? body.asOfColumn.trim() : null;
|
|
73746
|
+
if (!fileId) {
|
|
73747
|
+
sendJson(res, { error: "fileId is required" }, 400);
|
|
73748
|
+
return true;
|
|
73749
|
+
}
|
|
73750
|
+
res.writeHead(200, {
|
|
73751
|
+
"content-type": "application/x-ndjson; charset=utf-8",
|
|
73752
|
+
"cache-control": "no-store"
|
|
73753
|
+
});
|
|
73754
|
+
const emit = (p3) => {
|
|
73755
|
+
res.write(JSON.stringify(p3) + "\n");
|
|
73756
|
+
};
|
|
73757
|
+
try {
|
|
73758
|
+
emit({ phase: "parse", message: "Reading source\u2026" });
|
|
73759
|
+
const data = await readImportSourceFromFile(deps.db, fileId, deps.latticeRoot);
|
|
73760
|
+
emit({ phase: "infer", message: "Analyzing schema\u2026" });
|
|
73761
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
73762
|
+
inferSchema(data),
|
|
73763
|
+
data
|
|
73764
|
+
);
|
|
73765
|
+
emit({
|
|
73766
|
+
phase: "infer",
|
|
73767
|
+
message: `Found ${String(inferredPlan.entities.length)} entities, ${String(inferredPlan.dimensions.length)} dimensions, ${String(inferredPlan.linkages.length)} links`
|
|
73768
|
+
});
|
|
73769
|
+
const match = matchSchemaToExisting(existingDataTables2(deps.db), inferredPlan);
|
|
73770
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, match.rename);
|
|
73771
|
+
if (views.length > 0) {
|
|
73772
|
+
emit({
|
|
73773
|
+
phase: "detect",
|
|
73774
|
+
message: `Detected ${String(views.length)} reconstructable views (no duplicated rows)`
|
|
73775
|
+
});
|
|
73776
|
+
}
|
|
73777
|
+
if (match.isKnownDocument) {
|
|
73778
|
+
emit({
|
|
73779
|
+
phase: "detect",
|
|
73780
|
+
message: `Recognized as a new period of an existing document \u2014 ${String(match.matchedCount)} of ${String(match.totalEntities)} tables matched`
|
|
73781
|
+
});
|
|
73782
|
+
}
|
|
73783
|
+
if (asOfColumn) {
|
|
73784
|
+
emit({ phase: "infer", message: `Dating each row by its "${asOfColumn}" column` });
|
|
73785
|
+
} else if (asOf) {
|
|
73786
|
+
emit({ phase: "infer", message: `Importing as a snapshot dated ${asOf}` });
|
|
73787
|
+
}
|
|
73788
|
+
const result = await materializeImport(
|
|
73789
|
+
{ db: deps.db, configPath: deps.configPath },
|
|
73790
|
+
data,
|
|
73791
|
+
plan,
|
|
73792
|
+
views,
|
|
73793
|
+
{
|
|
73794
|
+
mode,
|
|
73795
|
+
asOf,
|
|
73796
|
+
asOfColumn,
|
|
73797
|
+
onProgress: async (p3) => {
|
|
73798
|
+
emit({ ...p3 });
|
|
73799
|
+
await new Promise((r6) => setImmediate(r6));
|
|
73800
|
+
}
|
|
73801
|
+
}
|
|
73802
|
+
);
|
|
73803
|
+
for (const t8 of result.tablesCreated) {
|
|
73804
|
+
deps.validTables.add(t8);
|
|
73805
|
+
const cols = deps.db.getRegisteredColumns(t8);
|
|
73806
|
+
if (cols && "deleted_at" in cols) deps.softDeletable.add(t8);
|
|
73807
|
+
}
|
|
73808
|
+
emit({ phase: "done", ok: true, result });
|
|
73809
|
+
} catch (e6) {
|
|
73810
|
+
emit({ phase: "error", message: e6.message });
|
|
73811
|
+
}
|
|
73812
|
+
res.end();
|
|
73813
|
+
return true;
|
|
73814
|
+
}
|
|
73815
|
+
|
|
72081
73816
|
// src/gui/read-routes.ts
|
|
72082
73817
|
init_http();
|
|
72083
73818
|
init_data();
|
|
@@ -72366,7 +74101,13 @@ async function handleReadRoutes(req, res, ctx, deps) {
|
|
|
72366
74101
|
return true;
|
|
72367
74102
|
}
|
|
72368
74103
|
if (method === "GET" && pathname === "/api/history") {
|
|
72369
|
-
const
|
|
74104
|
+
const limitRaw = url.searchParams.get("limit");
|
|
74105
|
+
const parsedLimit = parsePageParam(limitRaw, "limit");
|
|
74106
|
+
if (parsedLimit === "invalid") {
|
|
74107
|
+
sendJson(res, { error: "limit must be a non-negative integer" }, 400);
|
|
74108
|
+
return true;
|
|
74109
|
+
}
|
|
74110
|
+
const limit = limitRaw === null ? 200 : parsedLimit;
|
|
72370
74111
|
const filterTable = url.searchParams.get("table");
|
|
72371
74112
|
const raw = await active.db.query("_lattice_gui_audit", { limit });
|
|
72372
74113
|
let entries = raw.map(parseAudit).sort((a6, b6) => b6.ts.localeCompare(a6.ts));
|
|
@@ -73410,8 +75151,8 @@ async function handleHistoryRoutes(req, res, ctx, deps) {
|
|
|
73410
75151
|
|
|
73411
75152
|
// src/gui/workspaces-routes.ts
|
|
73412
75153
|
init_http();
|
|
73413
|
-
import { resolve as
|
|
73414
|
-
import { existsSync as
|
|
75154
|
+
import { resolve as resolve12 } from "path";
|
|
75155
|
+
import { existsSync as existsSync28, rmSync } from "fs";
|
|
73415
75156
|
init_workspace();
|
|
73416
75157
|
init_lattice_root();
|
|
73417
75158
|
init_user_config();
|
|
@@ -73419,7 +75160,7 @@ function cleanupWorkspaceFiles(root6, ws) {
|
|
|
73419
75160
|
if (!ws.configPath && ws.kind === "local") {
|
|
73420
75161
|
rmSync(workspaceDir(root6, ws.dir), { recursive: true, force: true });
|
|
73421
75162
|
} else if (ws.kind === "cloud") {
|
|
73422
|
-
if (ws.configPath &&
|
|
75163
|
+
if (ws.configPath && existsSync28(ws.configPath)) {
|
|
73423
75164
|
rmSync(ws.configPath, { force: true });
|
|
73424
75165
|
}
|
|
73425
75166
|
const labelMatch = /^\$\{LATTICE_DB:([A-Za-z0-9._-]+)\}$/.exec(ws.db.trim());
|
|
@@ -73573,7 +75314,7 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
73573
75314
|
return true;
|
|
73574
75315
|
}
|
|
73575
75316
|
const wsPaths = resolveWorkspacePaths(latticeRoot, ws);
|
|
73576
|
-
const isActive =
|
|
75317
|
+
const isActive = resolve12(active.configPath) === resolve12(wsPaths.configPath);
|
|
73577
75318
|
let switchedTo = null;
|
|
73578
75319
|
if (isActive) {
|
|
73579
75320
|
const fallback = listWorkspaces(latticeRoot).find((w2) => w2.id !== ws.id);
|
|
@@ -73622,40 +75363,40 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
73622
75363
|
|
|
73623
75364
|
// src/gui/databases-routes.ts
|
|
73624
75365
|
init_http();
|
|
73625
|
-
import { basename as
|
|
73626
|
-
import { existsSync as
|
|
75366
|
+
import { basename as basename13, resolve as resolve14 } from "path";
|
|
75367
|
+
import { existsSync as existsSync30 } from "fs";
|
|
73627
75368
|
init_parser();
|
|
73628
75369
|
|
|
73629
75370
|
// src/gui/config-paths.ts
|
|
73630
75371
|
init_parser();
|
|
73631
|
-
import { basename as
|
|
75372
|
+
import { basename as basename12, dirname as dirname16, join as join30, resolve as resolve13 } from "path";
|
|
73632
75373
|
import {
|
|
73633
|
-
existsSync as
|
|
75374
|
+
existsSync as existsSync29,
|
|
73634
75375
|
mkdirSync as mkdirSync12,
|
|
73635
|
-
readFileSync as
|
|
75376
|
+
readFileSync as readFileSync24,
|
|
73636
75377
|
readdirSync as readdirSync8,
|
|
73637
75378
|
unlinkSync as unlinkSync5,
|
|
73638
75379
|
writeFileSync as writeFileSync10
|
|
73639
75380
|
} from "fs";
|
|
73640
75381
|
import { parseDocument as parseDocument7 } from "yaml";
|
|
73641
75382
|
function resolveOutputDirForConfig(configPath) {
|
|
73642
|
-
const base = dirname16(
|
|
75383
|
+
const base = dirname16(resolve13(configPath));
|
|
73643
75384
|
for (const dir of ["context", ".", "generated"]) {
|
|
73644
|
-
const abs =
|
|
73645
|
-
if (
|
|
75385
|
+
const abs = resolve13(base, dir);
|
|
75386
|
+
if (existsSync29(join30(abs, ".lattice", "manifest.json"))) return abs;
|
|
73646
75387
|
}
|
|
73647
|
-
return
|
|
75388
|
+
return resolve13(base, "context");
|
|
73648
75389
|
}
|
|
73649
75390
|
function friendlyConfigName(parsedName, configPath) {
|
|
73650
75391
|
if (parsedName && parsedName.trim().length > 0) return parsedName.trim();
|
|
73651
|
-
return
|
|
75392
|
+
return basename12(configPath).replace(/\.(ya?ml)$/, "");
|
|
73652
75393
|
}
|
|
73653
75394
|
function listConfigs(activeConfigPath) {
|
|
73654
75395
|
const dir = dirname16(activeConfigPath);
|
|
73655
75396
|
const entries = [];
|
|
73656
75397
|
for (const fname of readdirSync8(dir)) {
|
|
73657
75398
|
if (!fname.endsWith(".yml") && !fname.endsWith(".yaml")) continue;
|
|
73658
|
-
const full =
|
|
75399
|
+
const full = join30(dir, fname);
|
|
73659
75400
|
try {
|
|
73660
75401
|
const parsed = parseConfigFile(full);
|
|
73661
75402
|
entries.push({
|
|
@@ -73666,7 +75407,7 @@ function listConfigs(activeConfigPath) {
|
|
|
73666
75407
|
// `label` is the friendly DB name — what the user sees in the
|
|
73667
75408
|
// dropdown + settings. Falls back to the basename when unset.
|
|
73668
75409
|
label: friendlyConfigName(parsed.name, full),
|
|
73669
|
-
dbFile:
|
|
75410
|
+
dbFile: basename12(parsed.dbPath),
|
|
73670
75411
|
active: full === activeConfigPath,
|
|
73671
75412
|
// `${LATTICE_DB:...}` and postgres:// configs resolve to a
|
|
73672
75413
|
// postgres URL; everything else is a local SQLite file. This
|
|
@@ -73683,37 +75424,37 @@ function createBlankConfig(activeConfigPath, dbName) {
|
|
|
73683
75424
|
const dir = dirname16(activeConfigPath);
|
|
73684
75425
|
const slug = dbName.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
73685
75426
|
if (!slug) throw new Error("Workspace name must contain at least one alphanumeric character");
|
|
73686
|
-
const configPath =
|
|
73687
|
-
if (
|
|
75427
|
+
const configPath = join30(dir, `${slug}.config.yml`);
|
|
75428
|
+
if (existsSync29(configPath)) throw new Error(`Config already exists: ${slug}.config.yml`);
|
|
73688
75429
|
const yaml = `db: ./data/${slug}.db
|
|
73689
75430
|
|
|
73690
75431
|
entities: {}
|
|
73691
75432
|
`;
|
|
73692
75433
|
writeFileSync10(configPath, yaml, "utf8");
|
|
73693
|
-
mkdirSync12(
|
|
75434
|
+
mkdirSync12(join30(dir, "data"), { recursive: true });
|
|
73694
75435
|
return configPath;
|
|
73695
75436
|
}
|
|
73696
75437
|
function sqliteFileForConfig(configPath) {
|
|
73697
|
-
const dbVal = parseDocument7(
|
|
75438
|
+
const dbVal = parseDocument7(readFileSync24(configPath, "utf8")).get("db");
|
|
73698
75439
|
const raw = (typeof dbVal === "string" ? dbVal : "").trim();
|
|
73699
75440
|
if (!raw) return null;
|
|
73700
75441
|
if (isPostgresUrl(raw) || raw.startsWith("${LATTICE_DB:")) return null;
|
|
73701
75442
|
if (raw === ":memory:" || raw.startsWith("file:")) return null;
|
|
73702
|
-
return
|
|
75443
|
+
return resolve13(dirname16(configPath), raw);
|
|
73703
75444
|
}
|
|
73704
75445
|
function deleteDatabaseFiles(targetConfigPath) {
|
|
73705
75446
|
const sqliteFile = sqliteFileForConfig(targetConfigPath);
|
|
73706
75447
|
unlinkSync5(targetConfigPath);
|
|
73707
75448
|
let deletedDbFile = null;
|
|
73708
|
-
if (sqliteFile &&
|
|
75449
|
+
if (sqliteFile && existsSync29(sqliteFile)) {
|
|
73709
75450
|
unlinkSync5(sqliteFile);
|
|
73710
75451
|
deletedDbFile = sqliteFile;
|
|
73711
75452
|
for (const suffix of ["-wal", "-shm", "-journal"]) {
|
|
73712
75453
|
const sidecar = sqliteFile + suffix;
|
|
73713
|
-
if (
|
|
75454
|
+
if (existsSync29(sidecar)) unlinkSync5(sidecar);
|
|
73714
75455
|
}
|
|
73715
75456
|
}
|
|
73716
|
-
return { deletedConfig:
|
|
75457
|
+
return { deletedConfig: basename12(targetConfigPath), deletedDbFile };
|
|
73717
75458
|
}
|
|
73718
75459
|
|
|
73719
75460
|
// src/gui/databases-routes.ts
|
|
@@ -73729,7 +75470,7 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
73729
75470
|
sendJson(res, {
|
|
73730
75471
|
current: {
|
|
73731
75472
|
path: active.configPath,
|
|
73732
|
-
dbFile:
|
|
75473
|
+
dbFile: basename13(parsedActive.dbPath),
|
|
73733
75474
|
label: friendlyLabel,
|
|
73734
75475
|
kind
|
|
73735
75476
|
},
|
|
@@ -73743,8 +75484,8 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
73743
75484
|
sendJson(res, { error: "path must be a string" }, 400);
|
|
73744
75485
|
return true;
|
|
73745
75486
|
}
|
|
73746
|
-
const newPath =
|
|
73747
|
-
if (!
|
|
75487
|
+
const newPath = resolve14(body.path);
|
|
75488
|
+
if (!existsSync30(newPath)) {
|
|
73748
75489
|
sendJson(res, { error: `Config not found: ${newPath}` }, 400);
|
|
73749
75490
|
return true;
|
|
73750
75491
|
}
|
|
@@ -73786,16 +75527,16 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
73786
75527
|
sendJson(res, { error: "path must be a non-empty string" }, 400);
|
|
73787
75528
|
return true;
|
|
73788
75529
|
}
|
|
73789
|
-
const target =
|
|
75530
|
+
const target = resolve14(body.path);
|
|
73790
75531
|
const known = listConfigs(active.configPath);
|
|
73791
|
-
const match = known.find((c6) =>
|
|
75532
|
+
const match = known.find((c6) => resolve14(c6.path) === target);
|
|
73792
75533
|
if (!match) {
|
|
73793
75534
|
sendJson(res, { error: `Not a known database config: ${target}` }, 400);
|
|
73794
75535
|
return true;
|
|
73795
75536
|
}
|
|
73796
75537
|
let switchedTo = null;
|
|
73797
|
-
if (
|
|
73798
|
-
const fallback = known.find((c6) =>
|
|
75538
|
+
if (resolve14(active.configPath) === target) {
|
|
75539
|
+
const fallback = known.find((c6) => resolve14(c6.path) !== target);
|
|
73799
75540
|
if (!fallback) {
|
|
73800
75541
|
sendJson(
|
|
73801
75542
|
res,
|
|
@@ -73886,20 +75627,26 @@ async function listenWithPortFallback(server, startPort, host) {
|
|
|
73886
75627
|
throw new Error(`No available port found starting at ${String(startPort)}`);
|
|
73887
75628
|
}
|
|
73888
75629
|
async function startGuiServer(options) {
|
|
73889
|
-
const bootConfigPath = options.configPath ?
|
|
73890
|
-
const bootOutputDir = options.outputDir ?
|
|
75630
|
+
const bootConfigPath = options.configPath ? resolve15(options.configPath) : null;
|
|
75631
|
+
const bootOutputDir = options.outputDir ? resolve15(options.outputDir) : null;
|
|
73891
75632
|
const startPort = options.port ?? 4317;
|
|
73892
75633
|
const host = options.host ?? "127.0.0.1";
|
|
75634
|
+
const isLoopbackHost2 = host === "localhost" || host === "::1" || host.startsWith("127.");
|
|
75635
|
+
if (!isLoopbackHost2) {
|
|
75636
|
+
console.warn(
|
|
75637
|
+
`[lattice] GUI is binding to a non-loopback address (${host}); its data routes are UNAUTHENTICATED and will be reachable from the network.`
|
|
75638
|
+
);
|
|
75639
|
+
}
|
|
73893
75640
|
const autoRender = options.autoRender ?? false;
|
|
73894
75641
|
const guiVersion = options.version ?? "";
|
|
73895
75642
|
const sessionId = crypto.randomUUID();
|
|
73896
75643
|
let updateService = null;
|
|
73897
75644
|
let activeRef = bootConfigPath && bootOutputDir ? await openConfig(bootConfigPath, bootOutputDir, autoRender, options.realtimeWatchdogMs) : null;
|
|
73898
|
-
const latticeRoot = (bootConfigPath ? findLatticeRoot(dirname17(bootConfigPath)) : null) ?? (options.latticeRoot ?
|
|
75645
|
+
const latticeRoot = (bootConfigPath ? findLatticeRoot(dirname17(bootConfigPath)) : null) ?? (options.latticeRoot ? resolve15(options.latticeRoot) : null);
|
|
73899
75646
|
let currentWorkspaceId = null;
|
|
73900
75647
|
if (latticeRoot && bootConfigPath) {
|
|
73901
75648
|
const launched = listWorkspaces(latticeRoot).find(
|
|
73902
|
-
(w2) =>
|
|
75649
|
+
(w2) => resolve15(resolveWorkspacePaths(latticeRoot, w2).configPath) === resolve15(bootConfigPath)
|
|
73903
75650
|
);
|
|
73904
75651
|
if (launched) {
|
|
73905
75652
|
currentWorkspaceId = launched.id;
|
|
@@ -74244,6 +75991,22 @@ async function startGuiServer(options) {
|
|
|
74244
75991
|
});
|
|
74245
75992
|
}
|
|
74246
75993
|
},
|
|
75994
|
+
// ── Structured-source import (apply) ──
|
|
75995
|
+
// The importer is reachable only via dropping a file in the assistant
|
|
75996
|
+
// chat; this materializes the user-confirmed proposal, re-reading the
|
|
75997
|
+
// file's bytes from its `fileId` (its retained blob).
|
|
75998
|
+
{
|
|
75999
|
+
handle: async (req2, res2) => {
|
|
76000
|
+
if (!pathname.startsWith("/api/import/")) return false;
|
|
76001
|
+
return await dispatchImportRoute(req2, res2, {
|
|
76002
|
+
db: active.db,
|
|
76003
|
+
configPath: active.configPath,
|
|
76004
|
+
latticeRoot: dirname17(active.configPath),
|
|
76005
|
+
validTables: active.validTables,
|
|
76006
|
+
softDeletable: active.softDeletable
|
|
76007
|
+
});
|
|
76008
|
+
}
|
|
76009
|
+
},
|
|
74247
76010
|
// ── Files: blob serving + open-in-finder ──
|
|
74248
76011
|
{
|
|
74249
76012
|
handle: async (req2, res2) => {
|
|
@@ -74434,6 +76197,7 @@ ${e6.stack ?? ""}`
|
|
|
74434
76197
|
server,
|
|
74435
76198
|
port,
|
|
74436
76199
|
url,
|
|
76200
|
+
whenConverged: () => activeRef?.converged ?? Promise.resolve(),
|
|
74437
76201
|
close: () => new Promise((resolveClose, reject) => {
|
|
74438
76202
|
updateService?.stop();
|
|
74439
76203
|
for (const client of wss.clients) {
|
|
@@ -74742,10 +76506,10 @@ function printHelp() {
|
|
|
74742
76506
|
);
|
|
74743
76507
|
}
|
|
74744
76508
|
function getVersion() {
|
|
74745
|
-
if (true) return "4.
|
|
76509
|
+
if (true) return "4.2.0";
|
|
74746
76510
|
try {
|
|
74747
76511
|
const pkgPath = new URL("../package.json", import.meta.url).pathname;
|
|
74748
|
-
const pkg = JSON.parse(
|
|
76512
|
+
const pkg = JSON.parse(readFileSync25(pkgPath, "utf-8"));
|
|
74749
76513
|
return pkg.version;
|
|
74750
76514
|
} catch {
|
|
74751
76515
|
return "unknown";
|
|
@@ -74776,10 +76540,10 @@ async function runUpdate() {
|
|
|
74776
76540
|
}
|
|
74777
76541
|
}
|
|
74778
76542
|
function runGenerate(args) {
|
|
74779
|
-
const configPath =
|
|
76543
|
+
const configPath = resolve16(args.config);
|
|
74780
76544
|
let raw;
|
|
74781
76545
|
try {
|
|
74782
|
-
raw =
|
|
76546
|
+
raw = readFileSync25(configPath, "utf-8");
|
|
74783
76547
|
} catch {
|
|
74784
76548
|
console.error(`Error: cannot read config file at "${configPath}"`);
|
|
74785
76549
|
process.exit(1);
|
|
@@ -74796,7 +76560,7 @@ function runGenerate(args) {
|
|
|
74796
76560
|
process.exit(1);
|
|
74797
76561
|
}
|
|
74798
76562
|
const configDir2 = dirname18(configPath);
|
|
74799
|
-
const outDir =
|
|
76563
|
+
const outDir = resolve16(args.out);
|
|
74800
76564
|
try {
|
|
74801
76565
|
const result = generateAll({ config, configDir: configDir2, outDir, scaffold: args.scaffold });
|
|
74802
76566
|
console.log(`Generated ${String(result.filesWritten.length)} file(s):`);
|
|
@@ -74809,8 +76573,8 @@ function runGenerate(args) {
|
|
|
74809
76573
|
}
|
|
74810
76574
|
}
|
|
74811
76575
|
async function runRender(args) {
|
|
74812
|
-
const outputDir =
|
|
74813
|
-
const configPath =
|
|
76576
|
+
const outputDir = resolve16(args.output);
|
|
76577
|
+
const configPath = resolve16(args.config);
|
|
74814
76578
|
let parsed;
|
|
74815
76579
|
try {
|
|
74816
76580
|
parsed = parseConfigFile(configPath);
|
|
@@ -74838,7 +76602,7 @@ async function runRender(args) {
|
|
|
74838
76602
|
}
|
|
74839
76603
|
}
|
|
74840
76604
|
async function runDoctor(args) {
|
|
74841
|
-
const db = new Lattice({ config:
|
|
76605
|
+
const db = new Lattice({ config: resolve16(args.config) });
|
|
74842
76606
|
try {
|
|
74843
76607
|
await db.init();
|
|
74844
76608
|
const report = await db.diagnoseRetrieval();
|
|
@@ -74865,7 +76629,7 @@ async function runSearch(args) {
|
|
|
74865
76629
|
console.error("Error: --table <table> is required for search");
|
|
74866
76630
|
process.exit(1);
|
|
74867
76631
|
}
|
|
74868
|
-
const db = new Lattice({ config:
|
|
76632
|
+
const db = new Lattice({ config: resolve16(args.config) });
|
|
74869
76633
|
try {
|
|
74870
76634
|
await db.init();
|
|
74871
76635
|
const results = await db.hybridSearch(args.table, args.query, { topK: args.topK ?? 10 });
|
|
@@ -74897,8 +76661,8 @@ async function runSearch(args) {
|
|
|
74897
76661
|
}
|
|
74898
76662
|
}
|
|
74899
76663
|
async function runReconcile(args, isDryRun) {
|
|
74900
|
-
const outputDir =
|
|
74901
|
-
const db = new Lattice({ config:
|
|
76664
|
+
const outputDir = resolve16(args.output);
|
|
76665
|
+
const db = new Lattice({ config: resolve16(args.config) });
|
|
74902
76666
|
try {
|
|
74903
76667
|
await db.init();
|
|
74904
76668
|
const start = Date.now();
|
|
@@ -74957,8 +76721,8 @@ function formatTimestamp() {
|
|
|
74957
76721
|
return `${hh}:${mm}:${ss}`;
|
|
74958
76722
|
}
|
|
74959
76723
|
async function runWatch(args) {
|
|
74960
|
-
const outputDir =
|
|
74961
|
-
const db = new Lattice({ config:
|
|
76724
|
+
const outputDir = resolve16(args.output);
|
|
76725
|
+
const db = new Lattice({ config: resolve16(args.config) });
|
|
74962
76726
|
try {
|
|
74963
76727
|
await db.init();
|
|
74964
76728
|
} catch (e6) {
|
|
@@ -75027,7 +76791,7 @@ async function runGui(args) {
|
|
|
75027
76791
|
if (args.root) process.env.LATTICE_ROOT = args.root;
|
|
75028
76792
|
const boot = ensureRootForGui({
|
|
75029
76793
|
startDir: args.root ?? process.cwd(),
|
|
75030
|
-
configPath:
|
|
76794
|
+
configPath: resolve16(args.config),
|
|
75031
76795
|
explicitConfig: args.config !== "./lattice.config.yml"
|
|
75032
76796
|
});
|
|
75033
76797
|
console.log(
|