latticesql 4.1.0 → 4.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/cli.js +2184 -309
- package/dist/index.cjs +2478 -571
- package/dist/index.d.cts +360 -1
- package/dist/index.d.ts +360 -1
- package/dist/index.js +2208 -316
- package/docs/api-reference.md +60 -4
- package/docs/architecture.md +24 -0
- package/docs/assistant.md +23 -0
- package/docs/configuration.md +27 -0
- package/docs/examples/dashboard.html +284 -0
- package/docs/importing.md +118 -0
- package/docs/retrieval.md +31 -0
- package/package.json +7 -3
package/dist/cli.js
CHANGED
|
@@ -205,10 +205,10 @@ function getOrCreateMasterKey() {
|
|
|
205
205
|
}
|
|
206
206
|
function readIdentity() {
|
|
207
207
|
const dir = ensureConfigDir();
|
|
208
|
-
const
|
|
209
|
-
if (!existsSync2(
|
|
208
|
+
const path3 = join2(dir, IDENTITY_FILENAME);
|
|
209
|
+
if (!existsSync2(path3)) return { ...EMPTY_IDENTITY };
|
|
210
210
|
try {
|
|
211
|
-
const parsed = JSON.parse(readFileSync(
|
|
211
|
+
const parsed = JSON.parse(readFileSync(path3, "utf8"));
|
|
212
212
|
return {
|
|
213
213
|
display_name: typeof parsed.display_name === "string" ? parsed.display_name : "",
|
|
214
214
|
email: typeof parsed.email === "string" ? parsed.email : ""
|
|
@@ -219,26 +219,26 @@ function readIdentity() {
|
|
|
219
219
|
}
|
|
220
220
|
function writeIdentity(identity) {
|
|
221
221
|
const dir = ensureConfigDir();
|
|
222
|
-
const
|
|
222
|
+
const path3 = join2(dir, IDENTITY_FILENAME);
|
|
223
223
|
const body = JSON.stringify(
|
|
224
224
|
{ display_name: identity.display_name, email: identity.email },
|
|
225
225
|
null,
|
|
226
226
|
2
|
|
227
227
|
);
|
|
228
|
-
writeFileSync(
|
|
228
|
+
writeFileSync(path3, body + "\n", "utf8");
|
|
229
229
|
if (platform2() !== "win32") {
|
|
230
230
|
try {
|
|
231
|
-
chmodSync2(
|
|
231
|
+
chmodSync2(path3, 384);
|
|
232
232
|
} catch {
|
|
233
233
|
}
|
|
234
234
|
}
|
|
235
235
|
}
|
|
236
236
|
function readPreferences() {
|
|
237
237
|
const dir = ensureConfigDir();
|
|
238
|
-
const
|
|
239
|
-
if (!existsSync2(
|
|
238
|
+
const path3 = join2(dir, PREFERENCES_FILENAME);
|
|
239
|
+
if (!existsSync2(path3)) return { ...DEFAULT_PREFERENCES };
|
|
240
240
|
try {
|
|
241
|
-
const parsed = JSON.parse(readFileSync(
|
|
241
|
+
const parsed = JSON.parse(readFileSync(path3, "utf8"));
|
|
242
242
|
const agg = typeof parsed.aggressiveness === "number" ? parsed.aggressiveness : NaN;
|
|
243
243
|
return {
|
|
244
244
|
show_system_tables: typeof parsed.show_system_tables === "boolean" ? parsed.show_system_tables : DEFAULT_PREFERENCES.show_system_tables,
|
|
@@ -252,7 +252,7 @@ function readPreferences() {
|
|
|
252
252
|
}
|
|
253
253
|
function writePreferences(prefs) {
|
|
254
254
|
const dir = ensureConfigDir();
|
|
255
|
-
const
|
|
255
|
+
const path3 = join2(dir, PREFERENCES_FILENAME);
|
|
256
256
|
const body = JSON.stringify(
|
|
257
257
|
{
|
|
258
258
|
show_system_tables: prefs.show_system_tables,
|
|
@@ -263,10 +263,10 @@ function writePreferences(prefs) {
|
|
|
263
263
|
null,
|
|
264
264
|
2
|
|
265
265
|
);
|
|
266
|
-
writeFileSync(
|
|
266
|
+
writeFileSync(path3, body + "\n", "utf8");
|
|
267
267
|
if (platform2() !== "win32") {
|
|
268
268
|
try {
|
|
269
|
-
chmodSync2(
|
|
269
|
+
chmodSync2(path3, 384);
|
|
270
270
|
} catch {
|
|
271
271
|
}
|
|
272
272
|
}
|
|
@@ -299,7 +299,9 @@ function withCredentialLock(fn) {
|
|
|
299
299
|
fd = openSync(lockPath, "wx");
|
|
300
300
|
break;
|
|
301
301
|
} catch (err) {
|
|
302
|
-
|
|
302
|
+
const code = err.code;
|
|
303
|
+
const contended = code === "EEXIST" || process.platform === "win32" && (code === "EPERM" || code === "EACCES");
|
|
304
|
+
if (!contended) throw err;
|
|
303
305
|
try {
|
|
304
306
|
if (Date.now() - statSync(lockPath).mtimeMs > LOCK_STALE_MS) {
|
|
305
307
|
unlinkSync(lockPath);
|
|
@@ -328,8 +330,8 @@ function withCredentialLock(fn) {
|
|
|
328
330
|
}
|
|
329
331
|
}
|
|
330
332
|
}
|
|
331
|
-
function writeFileAtomic(
|
|
332
|
-
const tmp = `${
|
|
333
|
+
function writeFileAtomic(path3, data) {
|
|
334
|
+
const tmp = `${path3}.${String(process.pid)}.${randomBytes2(4).toString("hex")}.tmp`;
|
|
333
335
|
writeFileSync(tmp, data, "utf8");
|
|
334
336
|
if (platform2() !== "win32") {
|
|
335
337
|
try {
|
|
@@ -337,7 +339,7 @@ function writeFileAtomic(path2, data) {
|
|
|
337
339
|
} catch {
|
|
338
340
|
}
|
|
339
341
|
}
|
|
340
|
-
renameSync(tmp,
|
|
342
|
+
renameSync(tmp, path3);
|
|
341
343
|
}
|
|
342
344
|
function mutateCredentials(mutate) {
|
|
343
345
|
withCredentialLock(() => {
|
|
@@ -348,11 +350,11 @@ function mutateCredentials(mutate) {
|
|
|
348
350
|
}
|
|
349
351
|
function loadCredentials() {
|
|
350
352
|
const dir = ensureConfigDir();
|
|
351
|
-
const
|
|
352
|
-
if (!existsSync2(
|
|
353
|
+
const path3 = join2(dir, DB_CREDENTIALS_FILENAME);
|
|
354
|
+
if (!existsSync2(path3)) return {};
|
|
353
355
|
const key = deriveKey(getOrCreateMasterKey());
|
|
354
356
|
try {
|
|
355
|
-
const ciphertext = readFileSync(
|
|
357
|
+
const ciphertext = readFileSync(path3, "utf8").trim();
|
|
356
358
|
const plaintext = decrypt(ciphertext, key);
|
|
357
359
|
const parsed = JSON.parse(plaintext);
|
|
358
360
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
@@ -367,10 +369,10 @@ function loadCredentials() {
|
|
|
367
369
|
}
|
|
368
370
|
function saveCredentials(creds) {
|
|
369
371
|
const dir = ensureConfigDir();
|
|
370
|
-
const
|
|
372
|
+
const path3 = join2(dir, DB_CREDENTIALS_FILENAME);
|
|
371
373
|
const key = deriveKey(getOrCreateMasterKey());
|
|
372
374
|
const ciphertext = encrypt(JSON.stringify(creds), key);
|
|
373
|
-
writeFileAtomic(
|
|
375
|
+
writeFileAtomic(path3, ciphertext + "\n");
|
|
374
376
|
}
|
|
375
377
|
function listDbCredentials() {
|
|
376
378
|
return Object.keys(loadCredentials()).sort();
|
|
@@ -418,11 +420,11 @@ function healRawDbUrl(configPath) {
|
|
|
418
420
|
}
|
|
419
421
|
function loadS3Configs() {
|
|
420
422
|
const dir = ensureConfigDir();
|
|
421
|
-
const
|
|
422
|
-
if (!existsSync2(
|
|
423
|
+
const path3 = join2(dir, S3_CONFIG_FILENAME);
|
|
424
|
+
if (!existsSync2(path3)) return {};
|
|
423
425
|
const key = deriveKey(getOrCreateMasterKey());
|
|
424
426
|
try {
|
|
425
|
-
const parsed = JSON.parse(decrypt(readFileSync(
|
|
427
|
+
const parsed = JSON.parse(decrypt(readFileSync(path3, "utf8").trim(), key));
|
|
426
428
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
427
429
|
return parsed;
|
|
428
430
|
}
|
|
@@ -436,12 +438,12 @@ function loadS3Configs() {
|
|
|
436
438
|
}
|
|
437
439
|
function saveS3Configs(cfgs) {
|
|
438
440
|
const dir = ensureConfigDir();
|
|
439
|
-
const
|
|
441
|
+
const path3 = join2(dir, S3_CONFIG_FILENAME);
|
|
440
442
|
const key = deriveKey(getOrCreateMasterKey());
|
|
441
|
-
writeFileSync(
|
|
443
|
+
writeFileSync(path3, encrypt(JSON.stringify(cfgs), key) + "\n", "utf8");
|
|
442
444
|
if (platform2() !== "win32") {
|
|
443
445
|
try {
|
|
444
|
-
chmodSync2(
|
|
446
|
+
chmodSync2(path3, 384);
|
|
445
447
|
} catch {
|
|
446
448
|
}
|
|
447
449
|
}
|
|
@@ -461,11 +463,11 @@ function deleteDbCredential(label) {
|
|
|
461
463
|
}
|
|
462
464
|
function loadAssistantCredentials() {
|
|
463
465
|
const dir = ensureConfigDir();
|
|
464
|
-
const
|
|
465
|
-
if (!existsSync2(
|
|
466
|
+
const path3 = join2(dir, ASSISTANT_CREDENTIALS_FILENAME);
|
|
467
|
+
if (!existsSync2(path3)) return {};
|
|
466
468
|
const key = deriveKey(getOrCreateMasterKey());
|
|
467
469
|
try {
|
|
468
|
-
const ciphertext = readFileSync(
|
|
470
|
+
const ciphertext = readFileSync(path3, "utf8").trim();
|
|
469
471
|
const plaintext = decrypt(ciphertext, key);
|
|
470
472
|
const parsed = JSON.parse(plaintext);
|
|
471
473
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
@@ -480,13 +482,13 @@ function loadAssistantCredentials() {
|
|
|
480
482
|
}
|
|
481
483
|
function saveAssistantCredentials(creds) {
|
|
482
484
|
const dir = ensureConfigDir();
|
|
483
|
-
const
|
|
485
|
+
const path3 = join2(dir, ASSISTANT_CREDENTIALS_FILENAME);
|
|
484
486
|
const key = deriveKey(getOrCreateMasterKey());
|
|
485
487
|
const ciphertext = encrypt(JSON.stringify(creds), key);
|
|
486
|
-
writeFileSync(
|
|
488
|
+
writeFileSync(path3, ciphertext + "\n", "utf8");
|
|
487
489
|
if (platform2() !== "win32") {
|
|
488
490
|
try {
|
|
489
|
-
chmodSync2(
|
|
491
|
+
chmodSync2(path3, 384);
|
|
490
492
|
} catch {
|
|
491
493
|
}
|
|
492
494
|
}
|
|
@@ -997,10 +999,10 @@ function manifestPath(outputDir) {
|
|
|
997
999
|
return join5(outputDir, ".lattice", "manifest.json");
|
|
998
1000
|
}
|
|
999
1001
|
function readManifest(outputDir) {
|
|
1000
|
-
const
|
|
1001
|
-
if (!existsSync5(
|
|
1002
|
+
const path3 = manifestPath(outputDir);
|
|
1003
|
+
if (!existsSync5(path3)) return null;
|
|
1002
1004
|
try {
|
|
1003
|
-
return JSON.parse(readFileSync4(
|
|
1005
|
+
return JSON.parse(readFileSync4(path3, "utf8"));
|
|
1004
1006
|
} catch {
|
|
1005
1007
|
return null;
|
|
1006
1008
|
}
|
|
@@ -1188,20 +1190,130 @@ var init_render_cursor = __esm({
|
|
|
1188
1190
|
}
|
|
1189
1191
|
});
|
|
1190
1192
|
|
|
1193
|
+
// src/db/load-sqlite.ts
|
|
1194
|
+
import path from "path";
|
|
1195
|
+
import { createRequire } from "module";
|
|
1196
|
+
import { spawnSync } from "child_process";
|
|
1197
|
+
function runtimeRequire() {
|
|
1198
|
+
const importMetaUrl = import.meta.url;
|
|
1199
|
+
return importMetaUrl ? createRequire(importMetaUrl) : (
|
|
1200
|
+
// CJS fallback — Node provides `require` on every CJS module scope. Under
|
|
1201
|
+
// tsup's CJS output `import.meta.url` is rewritten to undefined, so this
|
|
1202
|
+
// branch keeps the loader working in the published .cjs bundle.
|
|
1203
|
+
__require
|
|
1204
|
+
);
|
|
1205
|
+
}
|
|
1206
|
+
function asCtor(mod) {
|
|
1207
|
+
return mod.default ?? mod;
|
|
1208
|
+
}
|
|
1209
|
+
function isAbiMismatch(err) {
|
|
1210
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1211
|
+
const code = err.code;
|
|
1212
|
+
return message.includes("NODE_MODULE_VERSION") || code === "ERR_DLOPEN_FAILED" || message.includes("was compiled against a different Node.js version");
|
|
1213
|
+
}
|
|
1214
|
+
function autoRebuildDisabled() {
|
|
1215
|
+
const v2 = process.env.LATTICE_SQLITE_NO_AUTOREBUILD;
|
|
1216
|
+
if (!v2) return false;
|
|
1217
|
+
const normalized = v2.trim().toLowerCase();
|
|
1218
|
+
return normalized !== "" && normalized !== "0" && normalized !== "false" && normalized !== "no";
|
|
1219
|
+
}
|
|
1220
|
+
function installRootFor(req) {
|
|
1221
|
+
const pkgJsonPath = req.resolve("better-sqlite3/package.json");
|
|
1222
|
+
return path.resolve(path.dirname(pkgJsonPath), "..", "..");
|
|
1223
|
+
}
|
|
1224
|
+
function defaultRebuild(installRoot) {
|
|
1225
|
+
const npmBin = process.platform === "win32" ? "npm.cmd" : "npm";
|
|
1226
|
+
const res = spawnSync(npmBin, ["rebuild", "better-sqlite3"], {
|
|
1227
|
+
cwd: installRoot,
|
|
1228
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1229
|
+
encoding: "utf8",
|
|
1230
|
+
timeout: 5 * 60 * 1e3
|
|
1231
|
+
});
|
|
1232
|
+
if (res.error) {
|
|
1233
|
+
return { ok: false, reason: res.error.message };
|
|
1234
|
+
}
|
|
1235
|
+
if (res.status !== 0) {
|
|
1236
|
+
const stderr = res.stderr.trim();
|
|
1237
|
+
const tail = stderr ? stderr.slice(-300) : `npm rebuild exited with code ${String(res.status)}`;
|
|
1238
|
+
return { ok: false, reason: tail };
|
|
1239
|
+
}
|
|
1240
|
+
return { ok: true };
|
|
1241
|
+
}
|
|
1242
|
+
function resolveSqliteCtor(options = {}) {
|
|
1243
|
+
const req = options.require ?? runtimeRequire();
|
|
1244
|
+
const rebuild = options.rebuild ?? defaultRebuild;
|
|
1245
|
+
const resolveInstallRoot = options.installRoot ?? installRootFor;
|
|
1246
|
+
const log = options.log ?? ((msg) => process.stderr.write(msg + "\n"));
|
|
1247
|
+
let firstError;
|
|
1248
|
+
try {
|
|
1249
|
+
return asCtor(req("better-sqlite3"));
|
|
1250
|
+
} catch (err) {
|
|
1251
|
+
firstError = err;
|
|
1252
|
+
}
|
|
1253
|
+
if (!isAbiMismatch(firstError)) {
|
|
1254
|
+
throw new Error(PEER_DEP_MISSING_MESSAGE);
|
|
1255
|
+
}
|
|
1256
|
+
if (autoRebuildDisabled()) {
|
|
1257
|
+
throw new Error(
|
|
1258
|
+
rebuildFailedMessage("automatic rebuild is disabled (LATTICE_SQLITE_NO_AUTOREBUILD)")
|
|
1259
|
+
);
|
|
1260
|
+
}
|
|
1261
|
+
log("[latticesql] SQLite engine built for a different Node runtime \u2014 rebuilding better-sqlite3\u2026");
|
|
1262
|
+
let installRoot;
|
|
1263
|
+
try {
|
|
1264
|
+
installRoot = resolveInstallRoot(req);
|
|
1265
|
+
} catch (err) {
|
|
1266
|
+
throw new Error(
|
|
1267
|
+
rebuildFailedMessage(
|
|
1268
|
+
"could not locate the better-sqlite3 install root (" + (err instanceof Error ? err.message : String(err)) + ")"
|
|
1269
|
+
)
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
const outcome = rebuild(installRoot);
|
|
1273
|
+
if (!outcome.ok) {
|
|
1274
|
+
throw new Error(rebuildFailedMessage(outcome.reason));
|
|
1275
|
+
}
|
|
1276
|
+
try {
|
|
1277
|
+
return asCtor(req("better-sqlite3"));
|
|
1278
|
+
} catch (err) {
|
|
1279
|
+
throw new Error(
|
|
1280
|
+
rebuildFailedMessage(
|
|
1281
|
+
"the rebuilt module still failed to load (" + (err instanceof Error ? err.message : String(err)) + ")"
|
|
1282
|
+
)
|
|
1283
|
+
);
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
function rebuildFailedMessage(reason) {
|
|
1287
|
+
return "latticesql: the better-sqlite3 native module doesn\u2019t match this Node runtime and an automatic rebuild did not complete (" + reason + "). Run `npm rebuild better-sqlite3` (or reinstall) and retry.";
|
|
1288
|
+
}
|
|
1289
|
+
function loadSqlite() {
|
|
1290
|
+
if (_ctor) return _ctor;
|
|
1291
|
+
_ctor = resolveSqliteCtor();
|
|
1292
|
+
return _ctor;
|
|
1293
|
+
}
|
|
1294
|
+
var PEER_DEP_MISSING_MESSAGE, _ctor;
|
|
1295
|
+
var init_load_sqlite = __esm({
|
|
1296
|
+
"src/db/load-sqlite.ts"() {
|
|
1297
|
+
"use strict";
|
|
1298
|
+
PEER_DEP_MISSING_MESSAGE = "better-sqlite3 is a required peer dependency of latticesql \u2014 install it (npm install better-sqlite3).";
|
|
1299
|
+
_ctor = null;
|
|
1300
|
+
}
|
|
1301
|
+
});
|
|
1302
|
+
|
|
1191
1303
|
// src/db/sqlite.ts
|
|
1192
|
-
import Database from "better-sqlite3";
|
|
1193
1304
|
var SQLiteAdapter;
|
|
1194
1305
|
var init_sqlite = __esm({
|
|
1195
1306
|
"src/db/sqlite.ts"() {
|
|
1196
1307
|
"use strict";
|
|
1308
|
+
init_load_sqlite();
|
|
1197
1309
|
SQLiteAdapter = class {
|
|
1198
1310
|
dialect = "sqlite";
|
|
1199
1311
|
_db = null;
|
|
1200
1312
|
_path;
|
|
1201
1313
|
_wal;
|
|
1202
1314
|
_busyTimeout;
|
|
1203
|
-
constructor(
|
|
1204
|
-
this._path =
|
|
1315
|
+
constructor(path3, options) {
|
|
1316
|
+
this._path = path3;
|
|
1205
1317
|
this._wal = options?.wal ?? true;
|
|
1206
1318
|
this._busyTimeout = options?.busyTimeout ?? 5e3;
|
|
1207
1319
|
}
|
|
@@ -1210,7 +1322,8 @@ var init_sqlite = __esm({
|
|
|
1210
1322
|
return this._db;
|
|
1211
1323
|
}
|
|
1212
1324
|
open() {
|
|
1213
|
-
|
|
1325
|
+
const Ctor = loadSqlite();
|
|
1326
|
+
this._db = new Ctor(this._path);
|
|
1214
1327
|
this._db.pragma(`busy_timeout = ${this._busyTimeout.toString()}`);
|
|
1215
1328
|
if (this._wal) {
|
|
1216
1329
|
this._db.pragma("journal_mode = WAL");
|
|
@@ -1382,16 +1495,16 @@ var init_sqlite = __esm({
|
|
|
1382
1495
|
});
|
|
1383
1496
|
|
|
1384
1497
|
// src/db/postgres.ts
|
|
1385
|
-
import
|
|
1498
|
+
import path2 from "path";
|
|
1386
1499
|
import { fileURLToPath } from "url";
|
|
1387
|
-
import { createRequire } from "module";
|
|
1500
|
+
import { createRequire as createRequire2 } from "module";
|
|
1388
1501
|
function moduleContext() {
|
|
1389
1502
|
if (_moduleContext) return _moduleContext;
|
|
1390
1503
|
const importMetaUrl = import.meta.url;
|
|
1391
1504
|
if (importMetaUrl) {
|
|
1392
1505
|
_moduleContext = {
|
|
1393
|
-
dir:
|
|
1394
|
-
require:
|
|
1506
|
+
dir: path2.dirname(fileURLToPath(importMetaUrl)),
|
|
1507
|
+
require: createRequire2(importMetaUrl)
|
|
1395
1508
|
};
|
|
1396
1509
|
} else {
|
|
1397
1510
|
_moduleContext = { dir: __dirname, require: __require };
|
|
@@ -3320,14 +3433,14 @@ var init_core = __esm({
|
|
|
3320
3433
|
columnRef(f6) {
|
|
3321
3434
|
const col = `"${ident(f6.col)}"`;
|
|
3322
3435
|
if (f6.jsonPath === void 0) return { sql: col, params: [] };
|
|
3323
|
-
const
|
|
3436
|
+
const path3 = Array.isArray(f6.jsonPath) ? f6.jsonPath : [f6.jsonPath];
|
|
3324
3437
|
const numeric = isNumericComparison(f6);
|
|
3325
3438
|
if (this.adapter.dialect === "postgres") {
|
|
3326
3439
|
const extract2 = `((${col})::jsonb #>> ?::text[])`;
|
|
3327
3440
|
const sql2 = numeric ? `(${extract2})::numeric` : extract2;
|
|
3328
|
-
return { sql: sql2, params: [
|
|
3441
|
+
return { sql: sql2, params: [path3] };
|
|
3329
3442
|
}
|
|
3330
|
-
const jsonpath = `$.${
|
|
3443
|
+
const jsonpath = `$.${path3.join(".")}`;
|
|
3331
3444
|
const extract = `json_extract(${col}, ?)`;
|
|
3332
3445
|
const sql = numeric ? `CAST(${extract} AS REAL)` : extract;
|
|
3333
3446
|
return { sql, params: [jsonpath] };
|
|
@@ -5902,8 +6015,8 @@ var init_pipeline = __esm({
|
|
|
5902
6015
|
|
|
5903
6016
|
// src/render/interpolate.ts
|
|
5904
6017
|
function interpolate(template, row) {
|
|
5905
|
-
return template.replace(/\{\{([^}]+)\}\}/g, (_,
|
|
5906
|
-
const parts =
|
|
6018
|
+
return template.replace(/\{\{([^}]+)\}\}/g, (_, path3) => {
|
|
6019
|
+
const parts = path3.trim().split(".");
|
|
5907
6020
|
let val = row;
|
|
5908
6021
|
for (const part of parts) {
|
|
5909
6022
|
if (val == null || typeof val !== "object") return "";
|
|
@@ -6052,13 +6165,13 @@ function uniqueDirName(displayName, existing) {
|
|
|
6052
6165
|
}
|
|
6053
6166
|
}
|
|
6054
6167
|
function readRegistry(root6) {
|
|
6055
|
-
const
|
|
6056
|
-
if (!existsSync11(
|
|
6168
|
+
const path3 = registryPath(root6);
|
|
6169
|
+
if (!existsSync11(path3)) return { ...EMPTY_REGISTRY, workspaces: [] };
|
|
6057
6170
|
let parsed;
|
|
6058
6171
|
try {
|
|
6059
|
-
parsed = JSON.parse(readFileSync8(
|
|
6172
|
+
parsed = JSON.parse(readFileSync8(path3, "utf-8"));
|
|
6060
6173
|
} catch (e6) {
|
|
6061
|
-
throw new Error(`Lattice: corrupt workspace registry at "${
|
|
6174
|
+
throw new Error(`Lattice: corrupt workspace registry at "${path3}": ${e6.message}`);
|
|
6062
6175
|
}
|
|
6063
6176
|
const reg = parsed;
|
|
6064
6177
|
return {
|
|
@@ -6068,11 +6181,11 @@ function readRegistry(root6) {
|
|
|
6068
6181
|
};
|
|
6069
6182
|
}
|
|
6070
6183
|
function writeRegistry(root6, registry) {
|
|
6071
|
-
const
|
|
6072
|
-
const tmp = `${
|
|
6184
|
+
const path3 = registryPath(root6);
|
|
6185
|
+
const tmp = `${path3}.tmp-${String(process.pid)}`;
|
|
6073
6186
|
writeFileSync4(tmp, `${JSON.stringify(registry, null, 2)}
|
|
6074
6187
|
`, "utf-8");
|
|
6075
|
-
renameSync3(tmp,
|
|
6188
|
+
renameSync3(tmp, path3);
|
|
6076
6189
|
}
|
|
6077
6190
|
function listWorkspaces(root6) {
|
|
6078
6191
|
return readRegistry(root6).workspaces;
|
|
@@ -6277,6 +6390,7 @@ function deriveCanonicalContexts(tables) {
|
|
|
6277
6390
|
childrenOf.set(rel.table, list);
|
|
6278
6391
|
}
|
|
6279
6392
|
}
|
|
6393
|
+
const byName = new Map(tables.map((t8) => [t8.name, t8.definition]));
|
|
6280
6394
|
const out = [];
|
|
6281
6395
|
for (const { name, definition } of tables) {
|
|
6282
6396
|
const files = {};
|
|
@@ -6292,11 +6406,32 @@ function deriveCanonicalContexts(tables) {
|
|
|
6292
6406
|
};
|
|
6293
6407
|
}
|
|
6294
6408
|
for (const child of childrenOf.get(name) ?? []) {
|
|
6295
|
-
|
|
6296
|
-
|
|
6297
|
-
|
|
6298
|
-
|
|
6299
|
-
|
|
6409
|
+
const childDef = byName.get(child.table);
|
|
6410
|
+
const childBt = childDef ? belongsToRelations(childDef) : [];
|
|
6411
|
+
const [rel0, rel1] = childBt;
|
|
6412
|
+
if (childDef && rel0 && rel1 && isRenderJunction(childDef, childBt)) {
|
|
6413
|
+
const localRel = rel0.foreignKey === child.foreignKey ? rel0 : rel1;
|
|
6414
|
+
const remoteRel = localRel === rel0 ? rel1 : rel0;
|
|
6415
|
+
const fileKey = remoteRel.table === name ? `${child.table.toUpperCase()}__${remoteRel.foreignKey.toUpperCase()}.md` : `${remoteRel.table.toUpperCase()}.md`;
|
|
6416
|
+
files[fileKey] = {
|
|
6417
|
+
source: {
|
|
6418
|
+
type: "manyToMany",
|
|
6419
|
+
junctionTable: child.table,
|
|
6420
|
+
localKey: localRel.foreignKey,
|
|
6421
|
+
remoteKey: remoteRel.foreignKey,
|
|
6422
|
+
remoteTable: remoteRel.table,
|
|
6423
|
+
references: remoteRel.references ?? "id"
|
|
6424
|
+
},
|
|
6425
|
+
render: renderRelated(remoteRel.table),
|
|
6426
|
+
omitIfEmpty: true
|
|
6427
|
+
};
|
|
6428
|
+
} else {
|
|
6429
|
+
files[`${child.table.toUpperCase()}.md`] = {
|
|
6430
|
+
source: { type: "hasMany", table: child.table, foreignKey: child.foreignKey },
|
|
6431
|
+
render: renderRelated(child.table),
|
|
6432
|
+
omitIfEmpty: true
|
|
6433
|
+
};
|
|
6434
|
+
}
|
|
6300
6435
|
}
|
|
6301
6436
|
out.push({
|
|
6302
6437
|
table: name,
|
|
@@ -6309,6 +6444,15 @@ function deriveCanonicalContexts(tables) {
|
|
|
6309
6444
|
}
|
|
6310
6445
|
return out;
|
|
6311
6446
|
}
|
|
6447
|
+
function isRenderJunction(def, bt) {
|
|
6448
|
+
if (bt.length !== 2) return false;
|
|
6449
|
+
const fks = new Set(bt.map((r6) => r6.foreignKey));
|
|
6450
|
+
if (fks.size !== 2) return false;
|
|
6451
|
+
const pk = Array.isArray(def.primaryKey) ? def.primaryKey : def.primaryKey != null ? [def.primaryKey] : [];
|
|
6452
|
+
if (pk.length === 2 && pk.every((c6) => fks.has(c6))) return true;
|
|
6453
|
+
const SYSTEM2 = /* @__PURE__ */ new Set(["id", "created_at", "updated_at", "deleted_at"]);
|
|
6454
|
+
return Object.keys(def.columns).every((c6) => fks.has(c6) || SYSTEM2.has(c6));
|
|
6455
|
+
}
|
|
6312
6456
|
function belongsToRelations(def) {
|
|
6313
6457
|
return Object.values(def.relations ?? {}).filter(
|
|
6314
6458
|
(r6) => r6.type === "belongsTo"
|
|
@@ -6702,6 +6846,19 @@ var init_vector_index = __esm({
|
|
|
6702
6846
|
}
|
|
6703
6847
|
});
|
|
6704
6848
|
|
|
6849
|
+
// src/search/limits.ts
|
|
6850
|
+
function clampTopK(topK) {
|
|
6851
|
+
if (!Number.isFinite(topK)) return 1;
|
|
6852
|
+
return Math.min(Math.max(1, Math.floor(topK)), SEARCH_TOPK_MAX);
|
|
6853
|
+
}
|
|
6854
|
+
var SEARCH_TOPK_MAX;
|
|
6855
|
+
var init_limits = __esm({
|
|
6856
|
+
"src/search/limits.ts"() {
|
|
6857
|
+
"use strict";
|
|
6858
|
+
SEARCH_TOPK_MAX = 1e3;
|
|
6859
|
+
}
|
|
6860
|
+
});
|
|
6861
|
+
|
|
6705
6862
|
// src/search/embeddings.ts
|
|
6706
6863
|
async function ensureEmbeddingsTable(adapter) {
|
|
6707
6864
|
let cols = [];
|
|
@@ -6848,9 +7005,10 @@ function cosineSimilarity(a6, b6) {
|
|
|
6848
7005
|
}
|
|
6849
7006
|
async function searchByEmbedding(adapter, table, queryText, config, topK, minScore, pkColumn = "id") {
|
|
6850
7007
|
const queryVector = await config.embed(queryText);
|
|
7008
|
+
const k6 = clampTopK(topK);
|
|
6851
7009
|
let ranked;
|
|
6852
7010
|
if (await vectorIndexAvailable(adapter) && await hasVectorIndex(adapter, table)) {
|
|
6853
|
-
const hits = await searchVectorIndex(adapter, table, queryVector,
|
|
7011
|
+
const hits = await searchVectorIndex(adapter, table, queryVector, k6 * 4, minScore);
|
|
6854
7012
|
ranked = hits.map((h6) => ({
|
|
6855
7013
|
pk: h6.pk,
|
|
6856
7014
|
score: h6.score,
|
|
@@ -6858,7 +7016,7 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
6858
7016
|
content: h6.content
|
|
6859
7017
|
}));
|
|
6860
7018
|
} else {
|
|
6861
|
-
ranked = await scanChunks(adapter, table, queryVector, minScore);
|
|
7019
|
+
ranked = await scanChunks(adapter, table, queryVector, minScore, config.maxScanChunks);
|
|
6862
7020
|
}
|
|
6863
7021
|
const bestByRow = /* @__PURE__ */ new Map();
|
|
6864
7022
|
for (const r6 of ranked) {
|
|
@@ -6883,11 +7041,20 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
6883
7041
|
if (r6.content !== null) result.matchedContent = r6.content;
|
|
6884
7042
|
}
|
|
6885
7043
|
results.push(result);
|
|
6886
|
-
if (results.length >=
|
|
7044
|
+
if (results.length >= k6) break;
|
|
6887
7045
|
}
|
|
6888
7046
|
return results;
|
|
6889
7047
|
}
|
|
6890
|
-
async function scanChunks(adapter, table, queryVector, minScore) {
|
|
7048
|
+
async function scanChunks(adapter, table, queryVector, minScore, maxScanChunks) {
|
|
7049
|
+
if (maxScanChunks !== void 0) {
|
|
7050
|
+
const countRows = await allAsyncOrSync(
|
|
7051
|
+
adapter,
|
|
7052
|
+
`SELECT COUNT(*) AS n FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
7053
|
+
[table]
|
|
7054
|
+
);
|
|
7055
|
+
const n3 = Number(countRows[0]?.n ?? 0);
|
|
7056
|
+
if (n3 > maxScanChunks) throw new EmbeddingScanTooLargeError(table, n3, maxScanChunks);
|
|
7057
|
+
}
|
|
6891
7058
|
const stored = await allAsyncOrSync(
|
|
6892
7059
|
adapter,
|
|
6893
7060
|
`SELECT "row_pk", "chunk_index", "content", "embedding", "vec_dim" FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
@@ -6997,13 +7164,14 @@ async function refreshEmbeddings(adapter, table, config, pkColumn = "id", opts =
|
|
|
6997
7164
|
}
|
|
6998
7165
|
return { embedded, skipped, removed };
|
|
6999
7166
|
}
|
|
7000
|
-
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError;
|
|
7167
|
+
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError, EmbeddingScanTooLargeError;
|
|
7001
7168
|
var init_embeddings = __esm({
|
|
7002
7169
|
"src/search/embeddings.ts"() {
|
|
7003
7170
|
"use strict";
|
|
7004
7171
|
init_adapter();
|
|
7005
7172
|
init_chunking();
|
|
7006
7173
|
init_vector_index();
|
|
7174
|
+
init_limits();
|
|
7007
7175
|
EMBEDDINGS_TABLE = "_lattice_embeddings";
|
|
7008
7176
|
EmbeddingDimensionMismatchError = class extends Error {
|
|
7009
7177
|
constructor(table, expected, found) {
|
|
@@ -7016,6 +7184,17 @@ var init_embeddings = __esm({
|
|
|
7016
7184
|
this.name = "EmbeddingDimensionMismatchError";
|
|
7017
7185
|
}
|
|
7018
7186
|
};
|
|
7187
|
+
EmbeddingScanTooLargeError = class extends Error {
|
|
7188
|
+
constructor(table, found, limit) {
|
|
7189
|
+
super(
|
|
7190
|
+
`Embedding scan on "${table}" would read ${String(found)} stored chunk vectors, over the configured maxScanChunks of ${String(limit)}. Add a native vector index (pgvector) for this table or raise maxScanChunks \u2014 Lattice will not silently truncate the scan, which would return incomplete results.`
|
|
7191
|
+
);
|
|
7192
|
+
this.table = table;
|
|
7193
|
+
this.found = found;
|
|
7194
|
+
this.limit = limit;
|
|
7195
|
+
this.name = "EmbeddingScanTooLargeError";
|
|
7196
|
+
}
|
|
7197
|
+
};
|
|
7019
7198
|
}
|
|
7020
7199
|
});
|
|
7021
7200
|
|
|
@@ -7368,7 +7547,7 @@ async function fetchLiveRows2(adapter, table, ids, pkColumn) {
|
|
|
7368
7547
|
return out;
|
|
7369
7548
|
}
|
|
7370
7549
|
async function hybridSearch(adapter, table, query, opts = {}) {
|
|
7371
|
-
const topK = opts.topK ?? 10;
|
|
7550
|
+
const topK = clampTopK(opts.topK ?? 10);
|
|
7372
7551
|
const rrfK = opts.rrfK ?? 60;
|
|
7373
7552
|
const pool = opts.poolSize ?? Math.max(topK * 4, 20);
|
|
7374
7553
|
const pkColumn = opts.pkColumn ?? "id";
|
|
@@ -7469,6 +7648,7 @@ var init_hybrid = __esm({
|
|
|
7469
7648
|
init_fts();
|
|
7470
7649
|
init_ranking();
|
|
7471
7650
|
init_rerank();
|
|
7651
|
+
init_limits();
|
|
7472
7652
|
}
|
|
7473
7653
|
});
|
|
7474
7654
|
|
|
@@ -7725,18 +7905,18 @@ function computedColumnOrder(table, computed) {
|
|
|
7725
7905
|
const names = new Set(Object.keys(computed));
|
|
7726
7906
|
const order = [];
|
|
7727
7907
|
const state2 = /* @__PURE__ */ new Map();
|
|
7728
|
-
const visit = (name,
|
|
7908
|
+
const visit = (name, path3) => {
|
|
7729
7909
|
const st = state2.get(name);
|
|
7730
7910
|
if (st === "done") return;
|
|
7731
7911
|
if (st === "visiting") {
|
|
7732
|
-
const start =
|
|
7733
|
-
throw new ComputedColumnCycleError(table, [...
|
|
7912
|
+
const start = path3.indexOf(name);
|
|
7913
|
+
throw new ComputedColumnCycleError(table, [...path3.slice(start), name]);
|
|
7734
7914
|
}
|
|
7735
7915
|
state2.set(name, "visiting");
|
|
7736
7916
|
const spec = computed[name];
|
|
7737
7917
|
if (spec) {
|
|
7738
7918
|
for (const dep of spec.deps) {
|
|
7739
|
-
if (names.has(dep)) visit(dep, [...
|
|
7919
|
+
if (names.has(dep)) visit(dep, [...path3, name]);
|
|
7740
7920
|
}
|
|
7741
7921
|
}
|
|
7742
7922
|
state2.set(name, "done");
|
|
@@ -8196,6 +8376,26 @@ RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
|
8196
8376
|
);
|
|
8197
8377
|
$fn$;
|
|
8198
8378
|
|
|
8379
|
+
-- Delete-event visibility, decided from the PRE-DELETE snapshot the delete trigger
|
|
8380
|
+
-- captures (the live row + its ownership record are gone after a delete, so
|
|
8381
|
+
-- lattice_row_visible can't be used). Keyed on session_user, SECURITY DEFINER \u2014
|
|
8382
|
+
-- the same per-recipient gate. MUST MIRROR lattice_row_visible's rule: the row is
|
|
8383
|
+
-- visible iff this member owned it, OR it was 'everyone', OR it was 'custom' and
|
|
8384
|
+
-- this member was a grantee. A NULL owner snapshot (a legacy delete emitted before
|
|
8385
|
+
-- the snapshot columns, or a row with no ownership record) yields false \u2014 fail
|
|
8386
|
+
-- closed, never forward. (tests/integration assert this agrees with
|
|
8387
|
+
-- lattice_row_visible for all three visibility states \u2014 the no-drift guard.)
|
|
8388
|
+
CREATE OR REPLACE FUNCTION lattice_delete_visible(
|
|
8389
|
+
p_owner_role text, p_visibility text, p_grantees text[]
|
|
8390
|
+
)
|
|
8391
|
+
RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
8392
|
+
SELECT p_owner_role IS NOT NULL AND (
|
|
8393
|
+
p_owner_role = session_user
|
|
8394
|
+
OR p_visibility = 'everyone'
|
|
8395
|
+
OR (p_visibility = 'custom' AND session_user = ANY(COALESCE(p_grantees, ARRAY[]::text[])))
|
|
8396
|
+
);
|
|
8397
|
+
$fn$;
|
|
8398
|
+
|
|
8199
8399
|
-- Shared owner gate: raises unless the connected member owns (p_table, p_pk).
|
|
8200
8400
|
-- p_action is spliced into the message so every caller keeps its exact wording.
|
|
8201
8401
|
-- SECURITY DEFINER + session_user (never current_user), the cloud identity invariant.
|
|
@@ -8370,6 +8570,14 @@ CREATE TABLE IF NOT EXISTS "__lattice_changes" (
|
|
|
8370
8570
|
"created_at" timestamptz NOT NULL DEFAULT now()
|
|
8371
8571
|
);
|
|
8372
8572
|
|
|
8573
|
+
-- Pre-delete visibility snapshot columns (added to existing clouds via ADD COLUMN
|
|
8574
|
+
-- IF NOT EXISTS). A delete event carries the row's visibility AT DELETE TIME so the
|
|
8575
|
+
-- live fan-out can gate it per recipient even though the ownership record is gone.
|
|
8576
|
+
-- NULL on upserts.
|
|
8577
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_owner_role" text;
|
|
8578
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_visibility" text;
|
|
8579
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_grantees" text[];
|
|
8580
|
+
|
|
8373
8581
|
CREATE OR REPLACE FUNCTION lattice_notify_change() RETURNS trigger
|
|
8374
8582
|
LANGUAGE plpgsql AS $fn$
|
|
8375
8583
|
BEGIN
|
|
@@ -8379,7 +8587,10 @@ BEGIN
|
|
|
8379
8587
|
'pk', NEW."pk",
|
|
8380
8588
|
'op', NEW."op",
|
|
8381
8589
|
'owner_role', NEW."owner_role",
|
|
8382
|
-
'created_at', NEW."created_at"
|
|
8590
|
+
'created_at', NEW."created_at",
|
|
8591
|
+
'del_owner_role', NEW."del_owner_role",
|
|
8592
|
+
'del_visibility', NEW."del_visibility",
|
|
8593
|
+
'del_grantees', NEW."del_grantees"
|
|
8383
8594
|
)::text);
|
|
8384
8595
|
RETURN NEW;
|
|
8385
8596
|
END $fn$;
|
|
@@ -8585,10 +8796,22 @@ BEGIN
|
|
|
8585
8796
|
VALUES (${lit}, ${pkNew}, 'upsert', session_user);
|
|
8586
8797
|
RETURN NEW;
|
|
8587
8798
|
ELSIF TG_OP = 'DELETE' THEN
|
|
8799
|
+
-- Snapshot the row's visibility BEFORE the cascade removes its ownership +
|
|
8800
|
+
-- grant records, so the realtime fan-out can gate the delete event per
|
|
8801
|
+
-- recipient (the live predicate can't \u2014 these records are gone post-delete).
|
|
8802
|
+
-- The grantee list is captured here because the grant rows are deleted in the
|
|
8803
|
+
-- same statement below; after that the 'custom' audience is unrecoverable.
|
|
8804
|
+
INSERT INTO "__lattice_changes"
|
|
8805
|
+
("table_name","pk","op","owner_role","del_owner_role","del_visibility","del_grantees")
|
|
8806
|
+
VALUES (${lit}, ${pkOld}, 'delete', session_user,
|
|
8807
|
+
(SELECT o."owner_role" FROM "__lattice_owners" o
|
|
8808
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
8809
|
+
(SELECT o."visibility" FROM "__lattice_owners" o
|
|
8810
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
8811
|
+
COALESCE((SELECT array_agg(g."grantee_role") FROM "__lattice_row_grants" g
|
|
8812
|
+
WHERE g."table_name" = ${lit} AND g."pk" = ${pkOld}), ARRAY[]::text[]));
|
|
8588
8813
|
DELETE FROM "__lattice_owners" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8589
8814
|
DELETE FROM "__lattice_row_grants" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8590
|
-
INSERT INTO "__lattice_changes" ("table_name","pk","op","owner_role")
|
|
8591
|
-
VALUES (${lit}, ${pkOld}, 'delete', session_user);
|
|
8592
8815
|
RETURN OLD;
|
|
8593
8816
|
END IF;
|
|
8594
8817
|
RETURN NEW;
|
|
@@ -11830,7 +12053,7 @@ function parsePageParam(raw, kind) {
|
|
|
11830
12053
|
}
|
|
11831
12054
|
function readJson(req, opts = {}) {
|
|
11832
12055
|
const maxBytes = opts.maxBytes ?? DEFAULT_BODY_MAX_BYTES;
|
|
11833
|
-
return new Promise((
|
|
12056
|
+
return new Promise((resolve17, reject) => {
|
|
11834
12057
|
let raw = "";
|
|
11835
12058
|
let overflowed = false;
|
|
11836
12059
|
req.setEncoding("utf8");
|
|
@@ -11846,7 +12069,7 @@ function readJson(req, opts = {}) {
|
|
|
11846
12069
|
req.on("end", () => {
|
|
11847
12070
|
if (overflowed) return;
|
|
11848
12071
|
try {
|
|
11849
|
-
|
|
12072
|
+
resolve17(raw ? JSON.parse(raw) : {});
|
|
11850
12073
|
} catch {
|
|
11851
12074
|
reject(new Error("Invalid JSON body"));
|
|
11852
12075
|
}
|
|
@@ -11866,11 +12089,12 @@ ${err.stack ?? ""}`);
|
|
|
11866
12089
|
sendJson(res, { error: err.message }, status);
|
|
11867
12090
|
}
|
|
11868
12091
|
}
|
|
11869
|
-
var DEFAULT_BODY_MAX_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
12092
|
+
var DEFAULT_BODY_MAX_BYTES, MAX_INGEST_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
11870
12093
|
var init_http = __esm({
|
|
11871
12094
|
"src/gui/http.ts"() {
|
|
11872
12095
|
"use strict";
|
|
11873
12096
|
DEFAULT_BODY_MAX_BYTES = 1e6;
|
|
12097
|
+
MAX_INGEST_BYTES = 5e7;
|
|
11874
12098
|
MAX_ROWS_PAGE = 1e3;
|
|
11875
12099
|
DEFAULT_ROWS_PAGE = 500;
|
|
11876
12100
|
BodyTooLargeError = class extends Error {
|
|
@@ -13430,7 +13654,7 @@ var init_oauth = __esm({
|
|
|
13430
13654
|
|
|
13431
13655
|
// src/gui/assistant-routes.ts
|
|
13432
13656
|
function readBuffer(req, maxBytes = 25e6) {
|
|
13433
|
-
return new Promise((
|
|
13657
|
+
return new Promise((resolve17, reject) => {
|
|
13434
13658
|
const chunks = [];
|
|
13435
13659
|
let size = 0;
|
|
13436
13660
|
req.on("data", (c6) => {
|
|
@@ -13439,7 +13663,7 @@ function readBuffer(req, maxBytes = 25e6) {
|
|
|
13439
13663
|
else chunks.push(c6);
|
|
13440
13664
|
});
|
|
13441
13665
|
req.on("end", () => {
|
|
13442
|
-
|
|
13666
|
+
resolve17(Buffer.concat(chunks));
|
|
13443
13667
|
});
|
|
13444
13668
|
req.on("error", reject);
|
|
13445
13669
|
});
|
|
@@ -14740,7 +14964,7 @@ async function takeHostSlot(host, minIntervalMs = urlIngestConfig().hostMinInter
|
|
|
14740
14964
|
const earliest = Math.max(now2, hostNextAllowed.get(key) ?? 0);
|
|
14741
14965
|
hostNextAllowed.set(key, earliest + minIntervalMs);
|
|
14742
14966
|
const wait = earliest - now2;
|
|
14743
|
-
if (wait > 0) await new Promise((
|
|
14967
|
+
if (wait > 0) await new Promise((resolve17) => setTimeout(resolve17, wait));
|
|
14744
14968
|
}
|
|
14745
14969
|
var Semaphore, FetchBudget, sharedGate, hostNextAllowed;
|
|
14746
14970
|
var init_fetch_policy = __esm({
|
|
@@ -14756,7 +14980,7 @@ var init_fetch_policy = __esm({
|
|
|
14756
14980
|
if (this.permits > 0) {
|
|
14757
14981
|
this.permits -= 1;
|
|
14758
14982
|
} else {
|
|
14759
|
-
await new Promise((
|
|
14983
|
+
await new Promise((resolve17) => this.waiters.push(resolve17));
|
|
14760
14984
|
}
|
|
14761
14985
|
let released = false;
|
|
14762
14986
|
return () => {
|
|
@@ -14993,8 +15217,8 @@ function fileContentGroups(rows, fuzzy, threshold) {
|
|
|
14993
15217
|
const t8 = get(r6, "extracted_text");
|
|
14994
15218
|
return typeof t8 === "string" && t8.trim().length > 0;
|
|
14995
15219
|
}).map((r6) => {
|
|
14996
|
-
const
|
|
14997
|
-
const key = fuzzy ? "txt:" +
|
|
15220
|
+
const norm3 = normalizeText(get(r6, "extracted_text"));
|
|
15221
|
+
const key = fuzzy ? "txt:" + norm3.slice(0, 2e3) : "txt:" + createHash5("sha256").update(norm3).digest("hex");
|
|
14998
15222
|
return { id: String(get(r6, "id")), key, createdAt: cellStrOrNull(get(r6, "created_at")) };
|
|
14999
15223
|
});
|
|
15000
15224
|
const txtGroups = findDuplicateGroups(txtItems, {
|
|
@@ -15260,11 +15484,11 @@ function stripHtml(html) {
|
|
|
15260
15484
|
const text = decodeXmlEntities(stripTags(noScript));
|
|
15261
15485
|
return text.replace(/[ \t\f\r]+/g, " ").replace(/ *\n */g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
15262
15486
|
}
|
|
15263
|
-
async function unzip(
|
|
15487
|
+
async function unzip(path3) {
|
|
15264
15488
|
const fflate = await loadParser("fflate");
|
|
15265
15489
|
if (!fflate || typeof fflate.unzipSync !== "function") return null;
|
|
15266
15490
|
try {
|
|
15267
|
-
const buf = await readFile(
|
|
15491
|
+
const buf = await readFile(path3);
|
|
15268
15492
|
let total = 0;
|
|
15269
15493
|
return fflate.unzipSync(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength), {
|
|
15270
15494
|
filter: (file) => {
|
|
@@ -15293,35 +15517,35 @@ var init_helpers2 = __esm({
|
|
|
15293
15517
|
|
|
15294
15518
|
// src/gui/ai/doc/ooxml.ts
|
|
15295
15519
|
import { readFile as readFile2 } from "fs/promises";
|
|
15296
|
-
async function extractDocx(
|
|
15520
|
+
async function extractDocx(path3) {
|
|
15297
15521
|
const mod = await loadParser("mammoth");
|
|
15298
15522
|
const lib = mod?.default ?? mod;
|
|
15299
15523
|
if (!lib || typeof lib.extractRawText !== "function") return null;
|
|
15300
15524
|
try {
|
|
15301
|
-
const { value } = await lib.extractRawText({ path:
|
|
15525
|
+
const { value } = await lib.extractRawText({ path: path3 });
|
|
15302
15526
|
return nullIfEmpty(value);
|
|
15303
15527
|
} catch {
|
|
15304
15528
|
return null;
|
|
15305
15529
|
}
|
|
15306
15530
|
}
|
|
15307
|
-
async function extractDoc(
|
|
15531
|
+
async function extractDoc(path3) {
|
|
15308
15532
|
const mod = await loadParser(
|
|
15309
15533
|
"word-extractor"
|
|
15310
15534
|
);
|
|
15311
15535
|
const Ctor = mod && "default" in mod ? mod.default : mod;
|
|
15312
15536
|
if (typeof Ctor !== "function") return null;
|
|
15313
15537
|
try {
|
|
15314
|
-
const doc = await new Ctor().extract(
|
|
15538
|
+
const doc = await new Ctor().extract(path3);
|
|
15315
15539
|
return nullIfEmpty(doc.getBody());
|
|
15316
15540
|
} catch {
|
|
15317
15541
|
return null;
|
|
15318
15542
|
}
|
|
15319
15543
|
}
|
|
15320
|
-
async function extractPdf(
|
|
15544
|
+
async function extractPdf(path3) {
|
|
15321
15545
|
const unpdf = await loadParser("unpdf");
|
|
15322
15546
|
if (!unpdf || typeof unpdf.getDocumentProxy !== "function") return null;
|
|
15323
15547
|
try {
|
|
15324
|
-
const buf = await readFile2(
|
|
15548
|
+
const buf = await readFile2(path3);
|
|
15325
15549
|
const data = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
15326
15550
|
const text = await withTimeout(
|
|
15327
15551
|
(async () => {
|
|
@@ -15353,8 +15577,8 @@ function slideText(xml) {
|
|
|
15353
15577
|
}
|
|
15354
15578
|
return paras.join("\n");
|
|
15355
15579
|
}
|
|
15356
|
-
async function extractPptx(
|
|
15357
|
-
const entries = await unzip(
|
|
15580
|
+
async function extractPptx(path3) {
|
|
15581
|
+
const entries = await unzip(path3);
|
|
15358
15582
|
if (!entries) return null;
|
|
15359
15583
|
const slides = Object.keys(entries).filter((n3) => /^ppt\/slides\/slide\d+\.xml$/.test(n3)).sort((a6, b6) => partNumber(a6) - partNumber(b6));
|
|
15360
15584
|
if (slides.length === 0) return null;
|
|
@@ -15372,8 +15596,8 @@ async function extractPptx(path2) {
|
|
|
15372
15596
|
}
|
|
15373
15597
|
return nullIfEmpty(parts.join("\n\n"));
|
|
15374
15598
|
}
|
|
15375
|
-
async function extractXlsx(
|
|
15376
|
-
const entries = await unzip(
|
|
15599
|
+
async function extractXlsx(path3) {
|
|
15600
|
+
const entries = await unzip(path3);
|
|
15377
15601
|
if (!entries) return null;
|
|
15378
15602
|
const shared = [];
|
|
15379
15603
|
const ssBytes = entries["xl/sharedStrings.xml"];
|
|
@@ -15431,8 +15655,8 @@ function odfWhitespace(s2) {
|
|
|
15431
15655
|
function odfParagraph(inner) {
|
|
15432
15656
|
return decodeXmlEntities(stripTags(odfWhitespace(inner))).trim();
|
|
15433
15657
|
}
|
|
15434
|
-
async function extractOdfText(
|
|
15435
|
-
const entries = await unzip(
|
|
15658
|
+
async function extractOdfText(path3) {
|
|
15659
|
+
const entries = await unzip(path3);
|
|
15436
15660
|
if (!entries) return null;
|
|
15437
15661
|
const contentBytes = entries["content.xml"];
|
|
15438
15662
|
if (!contentBytes) return null;
|
|
@@ -15454,8 +15678,8 @@ async function extractOdfText(path2) {
|
|
|
15454
15678
|
}
|
|
15455
15679
|
return nullIfEmpty(lines.join("\n"));
|
|
15456
15680
|
}
|
|
15457
|
-
async function extractOds(
|
|
15458
|
-
const entries = await unzip(
|
|
15681
|
+
async function extractOds(path3) {
|
|
15682
|
+
const entries = await unzip(path3);
|
|
15459
15683
|
if (!entries) return null;
|
|
15460
15684
|
const contentBytes = entries["content.xml"];
|
|
15461
15685
|
if (!contentBytes) return null;
|
|
@@ -15514,8 +15738,8 @@ function resolveHref(baseDir, href) {
|
|
|
15514
15738
|
}
|
|
15515
15739
|
return normalizeZipPath(baseDir + h6);
|
|
15516
15740
|
}
|
|
15517
|
-
async function extractEpub(
|
|
15518
|
-
const entries = await unzip(
|
|
15741
|
+
async function extractEpub(path3) {
|
|
15742
|
+
const entries = await unzip(path3);
|
|
15519
15743
|
if (!entries) return null;
|
|
15520
15744
|
let order = [];
|
|
15521
15745
|
const container = entries["META-INF/container.xml"];
|
|
@@ -15599,9 +15823,9 @@ function rtfToText(rtf) {
|
|
|
15599
15823
|
s2 = s2.replace(/[{}]/g, "");
|
|
15600
15824
|
return s2.replace(/[ \t]+/g, (m4) => m4.includes(" ") ? " " : " ").replace(/[ \t]\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
15601
15825
|
}
|
|
15602
|
-
async function extractRtf(
|
|
15826
|
+
async function extractRtf(path3) {
|
|
15603
15827
|
try {
|
|
15604
|
-
const raw = await readFile3(
|
|
15828
|
+
const raw = await readFile3(path3, "latin1");
|
|
15605
15829
|
if (!raw.startsWith("{\\rtf")) return null;
|
|
15606
15830
|
return nullIfEmpty(rtfToText(raw));
|
|
15607
15831
|
} catch {
|
|
@@ -15647,27 +15871,27 @@ var init_other = __esm({
|
|
|
15647
15871
|
});
|
|
15648
15872
|
|
|
15649
15873
|
// src/gui/ai/doc-extractors.ts
|
|
15650
|
-
async function extractDocument(
|
|
15874
|
+
async function extractDocument(path3, ext) {
|
|
15651
15875
|
switch (ext) {
|
|
15652
15876
|
case ".docx":
|
|
15653
|
-
return extractDocx(
|
|
15877
|
+
return extractDocx(path3);
|
|
15654
15878
|
case ".doc":
|
|
15655
|
-
return extractDoc(
|
|
15879
|
+
return extractDoc(path3);
|
|
15656
15880
|
case ".pdf":
|
|
15657
|
-
return extractPdf(
|
|
15881
|
+
return extractPdf(path3);
|
|
15658
15882
|
case ".pptx":
|
|
15659
|
-
return extractPptx(
|
|
15883
|
+
return extractPptx(path3);
|
|
15660
15884
|
case ".xlsx":
|
|
15661
|
-
return extractXlsx(
|
|
15885
|
+
return extractXlsx(path3);
|
|
15662
15886
|
case ".odt":
|
|
15663
15887
|
case ".odp":
|
|
15664
|
-
return extractOdfText(
|
|
15888
|
+
return extractOdfText(path3);
|
|
15665
15889
|
case ".ods":
|
|
15666
|
-
return extractOds(
|
|
15890
|
+
return extractOds(path3);
|
|
15667
15891
|
case ".epub":
|
|
15668
|
-
return extractEpub(
|
|
15892
|
+
return extractEpub(path3);
|
|
15669
15893
|
case ".rtf":
|
|
15670
|
-
return extractRtf(
|
|
15894
|
+
return extractRtf(path3);
|
|
15671
15895
|
default:
|
|
15672
15896
|
return null;
|
|
15673
15897
|
}
|
|
@@ -15690,17 +15914,17 @@ function languageOf(name) {
|
|
|
15690
15914
|
function truncate(s2) {
|
|
15691
15915
|
return s2.length > MAX_TEXT2 ? s2.slice(0, MAX_TEXT2) : s2;
|
|
15692
15916
|
}
|
|
15693
|
-
async function parseFile(
|
|
15694
|
-
const name = originalName ?? basename4(
|
|
15917
|
+
async function parseFile(path3, mimeHint, originalName) {
|
|
15918
|
+
const name = originalName ?? basename4(path3);
|
|
15695
15919
|
const ext = extname(name).toLowerCase();
|
|
15696
15920
|
const lang = languageOf(name);
|
|
15697
15921
|
if (lang) {
|
|
15698
|
-
return { text: truncate(await readFile4(
|
|
15922
|
+
return { text: truncate(await readFile4(path3, "utf8")), language: lang };
|
|
15699
15923
|
}
|
|
15700
15924
|
if (mimeHint && TEXT_MIME.test(mimeHint) || TEXT_EXT.has(ext)) {
|
|
15701
|
-
return { text: truncate(await readFile4(
|
|
15925
|
+
return { text: truncate(await readFile4(path3, "utf8")) };
|
|
15702
15926
|
}
|
|
15703
|
-
const doc = await extractDocument(
|
|
15927
|
+
const doc = await extractDocument(path3, ext);
|
|
15704
15928
|
if (doc != null) {
|
|
15705
15929
|
return { text: truncate(doc) };
|
|
15706
15930
|
}
|
|
@@ -15770,7 +15994,7 @@ var init_extract = __esm({
|
|
|
15770
15994
|
});
|
|
15771
15995
|
|
|
15772
15996
|
// src/ai/llm-client.ts
|
|
15773
|
-
import { createRequire as
|
|
15997
|
+
import { createRequire as createRequire4 } from "module";
|
|
15774
15998
|
var DEFAULT_MODEL;
|
|
15775
15999
|
var init_llm_client = __esm({
|
|
15776
16000
|
"src/ai/llm-client.ts"() {
|
|
@@ -16324,7 +16548,7 @@ var init_url_safety = __esm({
|
|
|
16324
16548
|
import { JSDOM } from "jsdom";
|
|
16325
16549
|
import { Readability } from "@mozilla/readability";
|
|
16326
16550
|
import { basename as basename5 } from "path";
|
|
16327
|
-
import { createRequire as
|
|
16551
|
+
import { createRequire as createRequire5 } from "module";
|
|
16328
16552
|
async function crawlUrl(rawUrl, opts = {}) {
|
|
16329
16553
|
const u2 = await assertSafeUrl(rawUrl, opts.allowPrivate ?? false);
|
|
16330
16554
|
const fetchImpl = opts.fetcher ?? fetch;
|
|
@@ -16571,7 +16795,7 @@ async function renderViaPlaywright(url, timeoutMs, warnIfMissing = false) {
|
|
|
16571
16795
|
let chromium;
|
|
16572
16796
|
try {
|
|
16573
16797
|
const importMetaUrl = import.meta.url;
|
|
16574
|
-
const req = importMetaUrl ?
|
|
16798
|
+
const req = importMetaUrl ? createRequire5(importMetaUrl) : __require;
|
|
16575
16799
|
const pw = req("playwright");
|
|
16576
16800
|
chromium = pw.chromium;
|
|
16577
16801
|
} catch {
|
|
@@ -16743,8 +16967,8 @@ function isWriteConflict(e6) {
|
|
|
16743
16967
|
function normalizeUrl(s2) {
|
|
16744
16968
|
try {
|
|
16745
16969
|
const u2 = new URL(s2.trim());
|
|
16746
|
-
const
|
|
16747
|
-
return `${u2.protocol}//${u2.host.toLowerCase()}${
|
|
16970
|
+
const path3 = u2.pathname.replace(/\/+$/, "");
|
|
16971
|
+
return `${u2.protocol}//${u2.host.toLowerCase()}${path3}${u2.search}`;
|
|
16748
16972
|
} catch {
|
|
16749
16973
|
return null;
|
|
16750
16974
|
}
|
|
@@ -17502,7 +17726,7 @@ var init_tools = __esm({
|
|
|
17502
17726
|
});
|
|
17503
17727
|
|
|
17504
17728
|
// src/gui/ai/chat.ts
|
|
17505
|
-
import { createRequire as
|
|
17729
|
+
import { createRequire as createRequire6 } from "module";
|
|
17506
17730
|
function capToolResult(s2) {
|
|
17507
17731
|
if (s2.length <= MAX_TOOL_RESULT_CHARS) return s2;
|
|
17508
17732
|
if (s2.length > MAX_TOOL_RESULT_SKIP)
|
|
@@ -17735,7 +17959,7 @@ async function* runChat(opts) {
|
|
|
17735
17959
|
function loadSdk() {
|
|
17736
17960
|
if (!_sdk) {
|
|
17737
17961
|
const importMetaUrl = import.meta.url;
|
|
17738
|
-
const req = importMetaUrl ?
|
|
17962
|
+
const req = importMetaUrl ? createRequire6(importMetaUrl) : __require;
|
|
17739
17963
|
try {
|
|
17740
17964
|
_sdk = req("@anthropic-ai/sdk");
|
|
17741
17965
|
} catch (err) {
|
|
@@ -18972,7 +19196,7 @@ var init_sleep = __esm({
|
|
|
18972
19196
|
"node_modules/@smithy/core/dist-es/submodules/client/util-waiter/utils/sleep.js"() {
|
|
18973
19197
|
"use strict";
|
|
18974
19198
|
sleep = (seconds) => {
|
|
18975
|
-
return new Promise((
|
|
19199
|
+
return new Promise((resolve17) => setTimeout(resolve17, seconds * 1e3));
|
|
18976
19200
|
};
|
|
18977
19201
|
}
|
|
18978
19202
|
});
|
|
@@ -19141,8 +19365,8 @@ var init_createWaiter = __esm({
|
|
|
19141
19365
|
init_waiter2();
|
|
19142
19366
|
abortTimeout = (abortSignal) => {
|
|
19143
19367
|
let onAbort;
|
|
19144
|
-
const promise = new Promise((
|
|
19145
|
-
onAbort = () =>
|
|
19368
|
+
const promise = new Promise((resolve17) => {
|
|
19369
|
+
onAbort = () => resolve17({ state: WaiterState.ABORTED });
|
|
19146
19370
|
if (typeof abortSignal.addEventListener === "function") {
|
|
19147
19371
|
abortSignal.addEventListener("abort", onAbort);
|
|
19148
19372
|
} else {
|
|
@@ -21562,14 +21786,14 @@ var init_readFile = __esm({
|
|
|
21562
21786
|
"use strict";
|
|
21563
21787
|
filePromises = {};
|
|
21564
21788
|
fileIntercept = {};
|
|
21565
|
-
readFile6 = (
|
|
21566
|
-
if (fileIntercept[
|
|
21567
|
-
return fileIntercept[
|
|
21789
|
+
readFile6 = (path3, options) => {
|
|
21790
|
+
if (fileIntercept[path3] !== void 0) {
|
|
21791
|
+
return fileIntercept[path3];
|
|
21568
21792
|
}
|
|
21569
|
-
if (!filePromises[
|
|
21570
|
-
filePromises[
|
|
21793
|
+
if (!filePromises[path3] || options?.ignoreCache) {
|
|
21794
|
+
filePromises[path3] = fsReadFile(path3, "utf8");
|
|
21571
21795
|
}
|
|
21572
|
-
return filePromises[
|
|
21796
|
+
return filePromises[path3];
|
|
21573
21797
|
};
|
|
21574
21798
|
}
|
|
21575
21799
|
});
|
|
@@ -21687,8 +21911,8 @@ var init_externalDataInterceptor = __esm({
|
|
|
21687
21911
|
getFileRecord() {
|
|
21688
21912
|
return fileIntercept;
|
|
21689
21913
|
},
|
|
21690
|
-
interceptFile(
|
|
21691
|
-
fileIntercept[
|
|
21914
|
+
interceptFile(path3, contents) {
|
|
21915
|
+
fileIntercept[path3] = Promise.resolve(contents);
|
|
21692
21916
|
},
|
|
21693
21917
|
getTokenRecord() {
|
|
21694
21918
|
return tokenIntercept;
|
|
@@ -22022,13 +22246,13 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
22022
22246
|
}
|
|
22023
22247
|
return { hostname: "169.254.169.254", path: "/" };
|
|
22024
22248
|
};
|
|
22025
|
-
imdsHttpGet = async ({ hostname, path:
|
|
22249
|
+
imdsHttpGet = async ({ hostname, path: path3 }) => {
|
|
22026
22250
|
const { request } = await import("http");
|
|
22027
|
-
return new Promise((
|
|
22251
|
+
return new Promise((resolve17, reject) => {
|
|
22028
22252
|
const req = request({
|
|
22029
22253
|
method: "GET",
|
|
22030
22254
|
hostname: hostname.replace(/^\[(.+)]$/, "$1"),
|
|
22031
|
-
path:
|
|
22255
|
+
path: path3,
|
|
22032
22256
|
timeout: 1e3,
|
|
22033
22257
|
signal: AbortSignal.timeout(1e3)
|
|
22034
22258
|
});
|
|
@@ -22050,7 +22274,7 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
22050
22274
|
const chunks = [];
|
|
22051
22275
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
22052
22276
|
res.on("end", () => {
|
|
22053
|
-
|
|
22277
|
+
resolve17(Buffer.concat(chunks));
|
|
22054
22278
|
req.destroy();
|
|
22055
22279
|
});
|
|
22056
22280
|
});
|
|
@@ -22229,8 +22453,8 @@ var init_createConfigValueProvider = __esm({
|
|
|
22229
22453
|
return endpoint.url.href;
|
|
22230
22454
|
}
|
|
22231
22455
|
if ("hostname" in endpoint) {
|
|
22232
|
-
const { protocol, hostname, port, path:
|
|
22233
|
-
return `${protocol}//${hostname}${port ? ":" + port : ""}${
|
|
22456
|
+
const { protocol, hostname, port, path: path3 } = endpoint;
|
|
22457
|
+
return `${protocol}//${hostname}${port ? ":" + port : ""}${path3}`;
|
|
22234
22458
|
}
|
|
22235
22459
|
}
|
|
22236
22460
|
return endpoint;
|
|
@@ -22667,18 +22891,18 @@ var init_getAttrPathList = __esm({
|
|
|
22667
22891
|
"node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttrPathList.js"() {
|
|
22668
22892
|
"use strict";
|
|
22669
22893
|
init_types3();
|
|
22670
|
-
getAttrPathList = (
|
|
22671
|
-
const parts =
|
|
22894
|
+
getAttrPathList = (path3) => {
|
|
22895
|
+
const parts = path3.split(".");
|
|
22672
22896
|
const pathList = [];
|
|
22673
22897
|
for (const part of parts) {
|
|
22674
22898
|
const squareBracketIndex = part.indexOf("[");
|
|
22675
22899
|
if (squareBracketIndex !== -1) {
|
|
22676
22900
|
if (part.indexOf("]") !== part.length - 1) {
|
|
22677
|
-
throw new EndpointError(`Path: '${
|
|
22901
|
+
throw new EndpointError(`Path: '${path3}' does not end with ']'`);
|
|
22678
22902
|
}
|
|
22679
22903
|
const arrayIndex = part.slice(squareBracketIndex + 1, -1);
|
|
22680
22904
|
if (Number.isNaN(parseInt(arrayIndex))) {
|
|
22681
|
-
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${
|
|
22905
|
+
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${path3}'`);
|
|
22682
22906
|
}
|
|
22683
22907
|
if (squareBracketIndex !== 0) {
|
|
22684
22908
|
pathList.push(part.slice(0, squareBracketIndex));
|
|
@@ -22700,9 +22924,9 @@ var init_getAttr = __esm({
|
|
|
22700
22924
|
"use strict";
|
|
22701
22925
|
init_types3();
|
|
22702
22926
|
init_getAttrPathList();
|
|
22703
|
-
getAttr = (value,
|
|
22927
|
+
getAttr = (value, path3) => getAttrPathList(path3).reduce((acc, index) => {
|
|
22704
22928
|
if (typeof acc !== "object") {
|
|
22705
|
-
throw new EndpointError(`Index '${index}' in '${
|
|
22929
|
+
throw new EndpointError(`Index '${index}' in '${path3}' not found in '${JSON.stringify(value)}'`);
|
|
22706
22930
|
} else if (Array.isArray(acc)) {
|
|
22707
22931
|
const i6 = parseInt(index);
|
|
22708
22932
|
return acc[i6 < 0 ? acc.length + i6 : i6];
|
|
@@ -22768,8 +22992,8 @@ var init_parseURL = __esm({
|
|
|
22768
22992
|
return value;
|
|
22769
22993
|
}
|
|
22770
22994
|
if (typeof value === "object" && "hostname" in value) {
|
|
22771
|
-
const { hostname: hostname2, port, protocol: protocol2 = "", path:
|
|
22772
|
-
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${
|
|
22995
|
+
const { hostname: hostname2, port, protocol: protocol2 = "", path: path3 = "", query = {} } = value;
|
|
22996
|
+
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${path3}`);
|
|
22773
22997
|
url.search = Object.entries(query).map(([k6, v2]) => `${k6}=${v2}`).join("&");
|
|
22774
22998
|
return url;
|
|
22775
22999
|
}
|
|
@@ -23810,7 +24034,7 @@ async function collectStream(stream) {
|
|
|
23810
24034
|
return collected;
|
|
23811
24035
|
}
|
|
23812
24036
|
function readToBase64(blob) {
|
|
23813
|
-
return new Promise((
|
|
24037
|
+
return new Promise((resolve17, reject) => {
|
|
23814
24038
|
const reader = new FileReader();
|
|
23815
24039
|
reader.onloadend = () => {
|
|
23816
24040
|
if (reader.readyState !== 2) {
|
|
@@ -23819,7 +24043,7 @@ function readToBase64(blob) {
|
|
|
23819
24043
|
const result = reader.result ?? "";
|
|
23820
24044
|
const commaIndex = result.indexOf(",");
|
|
23821
24045
|
const dataOffset = commaIndex > -1 ? commaIndex + 1 : result.length;
|
|
23822
|
-
|
|
24046
|
+
resolve17(result.substring(dataOffset));
|
|
23823
24047
|
};
|
|
23824
24048
|
reader.onabort = () => reject(new Error("Read aborted"));
|
|
23825
24049
|
reader.onerror = () => reject(reader.error);
|
|
@@ -23947,7 +24171,7 @@ var init_stream_collector = __esm({
|
|
|
23947
24171
|
if (isReadableStreamInstance(stream)) {
|
|
23948
24172
|
return collectReadableStream(stream);
|
|
23949
24173
|
}
|
|
23950
|
-
return new Promise((
|
|
24174
|
+
return new Promise((resolve17, reject) => {
|
|
23951
24175
|
const collector = new Collector();
|
|
23952
24176
|
stream.pipe(collector);
|
|
23953
24177
|
stream.on("error", (err) => {
|
|
@@ -23957,7 +24181,7 @@ var init_stream_collector = __esm({
|
|
|
23957
24181
|
collector.on("error", reject);
|
|
23958
24182
|
collector.on("finish", function() {
|
|
23959
24183
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
23960
|
-
|
|
24184
|
+
resolve17(bytes);
|
|
23961
24185
|
});
|
|
23962
24186
|
});
|
|
23963
24187
|
};
|
|
@@ -24104,11 +24328,11 @@ var init_SerdeContext = __esm({
|
|
|
24104
24328
|
// node_modules/tslib/tslib.es6.mjs
|
|
24105
24329
|
function __awaiter(thisArg, _arguments, P2, generator) {
|
|
24106
24330
|
function adopt(value) {
|
|
24107
|
-
return value instanceof P2 ? value : new P2(function(
|
|
24108
|
-
|
|
24331
|
+
return value instanceof P2 ? value : new P2(function(resolve17) {
|
|
24332
|
+
resolve17(value);
|
|
24109
24333
|
});
|
|
24110
24334
|
}
|
|
24111
|
-
return new (P2 || (P2 = Promise))(function(
|
|
24335
|
+
return new (P2 || (P2 = Promise))(function(resolve17, reject) {
|
|
24112
24336
|
function fulfilled(value) {
|
|
24113
24337
|
try {
|
|
24114
24338
|
step(generator.next(value));
|
|
@@ -24124,7 +24348,7 @@ function __awaiter(thisArg, _arguments, P2, generator) {
|
|
|
24124
24348
|
}
|
|
24125
24349
|
}
|
|
24126
24350
|
function step(result) {
|
|
24127
|
-
result.done ?
|
|
24351
|
+
result.done ? resolve17(result.value) : adopt(result.value).then(fulfilled, rejected);
|
|
24128
24352
|
}
|
|
24129
24353
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
24130
24354
|
});
|
|
@@ -25321,7 +25545,7 @@ async function* readableToIterable(readStream) {
|
|
|
25321
25545
|
streamEnded = true;
|
|
25322
25546
|
});
|
|
25323
25547
|
while (!generationEnded) {
|
|
25324
|
-
const value = await new Promise((
|
|
25548
|
+
const value = await new Promise((resolve17) => setTimeout(() => resolve17(records.shift()), 0));
|
|
25325
25549
|
if (value) {
|
|
25326
25550
|
yield value;
|
|
25327
25551
|
}
|
|
@@ -25880,11 +26104,11 @@ var init_HttpBindingProtocol = __esm({
|
|
|
25880
26104
|
const opTraits = translateTraits(operationSchema.traits);
|
|
25881
26105
|
if (opTraits.http) {
|
|
25882
26106
|
request.method = opTraits.http[0];
|
|
25883
|
-
const [
|
|
26107
|
+
const [path3, search] = opTraits.http[1].split("?");
|
|
25884
26108
|
if (request.path == "/") {
|
|
25885
|
-
request.path =
|
|
26109
|
+
request.path = path3;
|
|
25886
26110
|
} else {
|
|
25887
|
-
request.path +=
|
|
26111
|
+
request.path += path3;
|
|
25888
26112
|
}
|
|
25889
26113
|
const traitSearchParams = new URLSearchParams(search ?? "");
|
|
25890
26114
|
for (const [key, value] of traitSearchParams) {
|
|
@@ -26866,7 +27090,7 @@ var init_retryMiddleware = __esm({
|
|
|
26866
27090
|
init_constants5();
|
|
26867
27091
|
init_parseRetryAfterHeader();
|
|
26868
27092
|
init_util2();
|
|
26869
|
-
cooldown = (ms) => new Promise((
|
|
27093
|
+
cooldown = (ms) => new Promise((resolve17) => setTimeout(resolve17, ms));
|
|
26870
27094
|
isRetryStrategyV2 = (retryStrategy) => typeof retryStrategy.acquireInitialRetryToken !== "undefined" && typeof retryStrategy.refreshRetryTokenForRetry !== "undefined" && typeof retryStrategy.recordSuccess !== "undefined";
|
|
26871
27095
|
getRetryErrorInfo = (error, logger2) => {
|
|
26872
27096
|
const errorInfo = {
|
|
@@ -26965,7 +27189,7 @@ var init_DefaultRateLimiter = __esm({
|
|
|
26965
27189
|
this.refillTokenBucket();
|
|
26966
27190
|
while (amount > this.availableTokens) {
|
|
26967
27191
|
const delay = (amount - this.availableTokens) / this.fillRate * 1e3;
|
|
26968
|
-
await new Promise((
|
|
27192
|
+
await new Promise((resolve17) => _DefaultRateLimiter.setTimeoutFn(resolve17, delay));
|
|
26969
27193
|
this.refillTokenBucket();
|
|
26970
27194
|
}
|
|
26971
27195
|
this.availableTokens = this.availableTokens - amount;
|
|
@@ -27863,9 +28087,9 @@ var init_createPaginator = __esm({
|
|
|
27863
28087
|
command = withCommand(command) ?? command;
|
|
27864
28088
|
return await client.send(command, ...args);
|
|
27865
28089
|
};
|
|
27866
|
-
get2 = (fromObject,
|
|
28090
|
+
get2 = (fromObject, path3) => {
|
|
27867
28091
|
let cursor = fromObject;
|
|
27868
|
-
const pathComponents =
|
|
28092
|
+
const pathComponents = path3.split(".");
|
|
27869
28093
|
for (const step of pathComponents) {
|
|
27870
28094
|
if (!cursor || typeof cursor !== "object") {
|
|
27871
28095
|
return void 0;
|
|
@@ -30357,10 +30581,10 @@ ${longDate}
|
|
|
30357
30581
|
${credentialScope}
|
|
30358
30582
|
${toHex(hashedRequest)}`;
|
|
30359
30583
|
}
|
|
30360
|
-
getCanonicalPath({ path:
|
|
30584
|
+
getCanonicalPath({ path: path3 }) {
|
|
30361
30585
|
if (this.uriEscapePath) {
|
|
30362
30586
|
const normalizedPathSegments = [];
|
|
30363
|
-
for (const pathSegment of
|
|
30587
|
+
for (const pathSegment of path3.split("/")) {
|
|
30364
30588
|
if (pathSegment?.length === 0)
|
|
30365
30589
|
continue;
|
|
30366
30590
|
if (pathSegment === ".")
|
|
@@ -30371,11 +30595,11 @@ ${toHex(hashedRequest)}`;
|
|
|
30371
30595
|
normalizedPathSegments.push(pathSegment);
|
|
30372
30596
|
}
|
|
30373
30597
|
}
|
|
30374
|
-
const normalizedPath = `${
|
|
30598
|
+
const normalizedPath = `${path3?.startsWith("/") ? "/" : ""}${normalizedPathSegments.join("/")}${normalizedPathSegments.length > 0 && path3?.endsWith("/") ? "/" : ""}`;
|
|
30375
30599
|
const doubleEncoded = escapeUri(normalizedPath);
|
|
30376
30600
|
return doubleEncoded.replace(/%2F/g, "/");
|
|
30377
30601
|
}
|
|
30378
|
-
return
|
|
30602
|
+
return path3;
|
|
30379
30603
|
}
|
|
30380
30604
|
validateResolvedCredentials(credentials) {
|
|
30381
30605
|
if (typeof credentials !== "object" || typeof credentials.accessKeyId !== "string" || typeof credentials.secretAccessKey !== "string") {
|
|
@@ -30635,8 +30859,8 @@ var init_SignatureV4 = __esm({
|
|
|
30635
30859
|
priorSignature: signableMessage.priorSignature,
|
|
30636
30860
|
eventStreamCredentials
|
|
30637
30861
|
});
|
|
30638
|
-
return promise.then((
|
|
30639
|
-
return { message: signableMessage.message, signature };
|
|
30862
|
+
return promise.then((signature2) => {
|
|
30863
|
+
return { message: signableMessage.message, signature: signature2 };
|
|
30640
30864
|
});
|
|
30641
30865
|
}
|
|
30642
30866
|
async signString(stringToSign, { signingDate = /* @__PURE__ */ new Date(), signingRegion, signingService, eventStreamCredentials } = {}) {
|
|
@@ -30664,8 +30888,8 @@ var init_SignatureV4 = __esm({
|
|
|
30664
30888
|
request.headers[SHA256_HEADER] = payloadHash;
|
|
30665
30889
|
}
|
|
30666
30890
|
const canonicalHeaders = getCanonicalHeaders(request, unsignableHeaders, signableHeaders);
|
|
30667
|
-
const
|
|
30668
|
-
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${
|
|
30891
|
+
const signature2 = await this.getSignature(longDate, scope, this.getSigningKey(credentials, region, shortDate, signingService), this.createCanonicalRequest(request, canonicalHeaders, payloadHash));
|
|
30892
|
+
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${signature2}`;
|
|
30669
30893
|
return request;
|
|
30670
30894
|
}
|
|
30671
30895
|
async getSignature(longDate, credentialScope, keyPromise, canonicalRequest) {
|
|
@@ -35292,16 +35516,16 @@ var init_Matcher = __esm({
|
|
|
35292
35516
|
* @returns {string|undefined}
|
|
35293
35517
|
*/
|
|
35294
35518
|
getCurrentTag() {
|
|
35295
|
-
const
|
|
35296
|
-
return
|
|
35519
|
+
const path3 = this._matcher.path;
|
|
35520
|
+
return path3.length > 0 ? path3[path3.length - 1].tag : void 0;
|
|
35297
35521
|
}
|
|
35298
35522
|
/**
|
|
35299
35523
|
* Get current namespace.
|
|
35300
35524
|
* @returns {string|undefined}
|
|
35301
35525
|
*/
|
|
35302
35526
|
getCurrentNamespace() {
|
|
35303
|
-
const
|
|
35304
|
-
return
|
|
35527
|
+
const path3 = this._matcher.path;
|
|
35528
|
+
return path3.length > 0 ? path3[path3.length - 1].namespace : void 0;
|
|
35305
35529
|
}
|
|
35306
35530
|
/**
|
|
35307
35531
|
* Get current node's attribute value.
|
|
@@ -35309,9 +35533,9 @@ var init_Matcher = __esm({
|
|
|
35309
35533
|
* @returns {*}
|
|
35310
35534
|
*/
|
|
35311
35535
|
getAttrValue(attrName) {
|
|
35312
|
-
const
|
|
35313
|
-
if (
|
|
35314
|
-
return
|
|
35536
|
+
const path3 = this._matcher.path;
|
|
35537
|
+
if (path3.length === 0) return void 0;
|
|
35538
|
+
return path3[path3.length - 1].values?.[attrName];
|
|
35315
35539
|
}
|
|
35316
35540
|
/**
|
|
35317
35541
|
* Check if current node has an attribute.
|
|
@@ -35319,9 +35543,9 @@ var init_Matcher = __esm({
|
|
|
35319
35543
|
* @returns {boolean}
|
|
35320
35544
|
*/
|
|
35321
35545
|
hasAttr(attrName) {
|
|
35322
|
-
const
|
|
35323
|
-
if (
|
|
35324
|
-
const current =
|
|
35546
|
+
const path3 = this._matcher.path;
|
|
35547
|
+
if (path3.length === 0) return false;
|
|
35548
|
+
const current = path3[path3.length - 1];
|
|
35325
35549
|
return current.values !== void 0 && attrName in current.values;
|
|
35326
35550
|
}
|
|
35327
35551
|
/**
|
|
@@ -35329,18 +35553,18 @@ var init_Matcher = __esm({
|
|
|
35329
35553
|
* @returns {number}
|
|
35330
35554
|
*/
|
|
35331
35555
|
getPosition() {
|
|
35332
|
-
const
|
|
35333
|
-
if (
|
|
35334
|
-
return
|
|
35556
|
+
const path3 = this._matcher.path;
|
|
35557
|
+
if (path3.length === 0) return -1;
|
|
35558
|
+
return path3[path3.length - 1].position ?? 0;
|
|
35335
35559
|
}
|
|
35336
35560
|
/**
|
|
35337
35561
|
* Get current node's repeat counter (occurrence count of this tag name).
|
|
35338
35562
|
* @returns {number}
|
|
35339
35563
|
*/
|
|
35340
35564
|
getCounter() {
|
|
35341
|
-
const
|
|
35342
|
-
if (
|
|
35343
|
-
return
|
|
35565
|
+
const path3 = this._matcher.path;
|
|
35566
|
+
if (path3.length === 0) return -1;
|
|
35567
|
+
return path3[path3.length - 1].counter ?? 0;
|
|
35344
35568
|
}
|
|
35345
35569
|
/**
|
|
35346
35570
|
* Get current node's sibling index (alias for getPosition).
|
|
@@ -46253,7 +46477,7 @@ var init_node_http = __esm({
|
|
|
46253
46477
|
|
|
46254
46478
|
// node_modules/@smithy/credential-provider-imds/dist-es/remoteProvider/httpRequest.js
|
|
46255
46479
|
function httpRequest(options) {
|
|
46256
|
-
return new Promise((
|
|
46480
|
+
return new Promise((resolve17, reject) => {
|
|
46257
46481
|
const req = node_http.request({
|
|
46258
46482
|
method: "GET",
|
|
46259
46483
|
...options,
|
|
@@ -46278,7 +46502,7 @@ function httpRequest(options) {
|
|
|
46278
46502
|
chunks.push(chunk);
|
|
46279
46503
|
});
|
|
46280
46504
|
res.on("end", () => {
|
|
46281
|
-
|
|
46505
|
+
resolve17(Buffer.concat(chunks));
|
|
46282
46506
|
req.destroy();
|
|
46283
46507
|
});
|
|
46284
46508
|
});
|
|
@@ -46923,21 +47147,21 @@ async function writeRequestBody(httpRequest2, request, maxContinueTimeoutMs = MI
|
|
|
46923
47147
|
let sendBody = true;
|
|
46924
47148
|
if (!externalAgent && expect === "100-continue") {
|
|
46925
47149
|
sendBody = await Promise.race([
|
|
46926
|
-
new Promise((
|
|
46927
|
-
timeoutId = Number(timing.setTimeout(() =>
|
|
47150
|
+
new Promise((resolve17) => {
|
|
47151
|
+
timeoutId = Number(timing.setTimeout(() => resolve17(true), Math.max(MIN_WAIT_TIME, maxContinueTimeoutMs)));
|
|
46928
47152
|
}),
|
|
46929
|
-
new Promise((
|
|
47153
|
+
new Promise((resolve17) => {
|
|
46930
47154
|
httpRequest2.on("continue", () => {
|
|
46931
47155
|
timing.clearTimeout(timeoutId);
|
|
46932
|
-
|
|
47156
|
+
resolve17(true);
|
|
46933
47157
|
});
|
|
46934
47158
|
httpRequest2.on("response", () => {
|
|
46935
47159
|
timing.clearTimeout(timeoutId);
|
|
46936
|
-
|
|
47160
|
+
resolve17(false);
|
|
46937
47161
|
});
|
|
46938
47162
|
httpRequest2.on("error", () => {
|
|
46939
47163
|
timing.clearTimeout(timeoutId);
|
|
46940
|
-
|
|
47164
|
+
resolve17(false);
|
|
46941
47165
|
});
|
|
46942
47166
|
})
|
|
46943
47167
|
]);
|
|
@@ -47035,13 +47259,13 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
47035
47259
|
return socketWarningTimestamp;
|
|
47036
47260
|
}
|
|
47037
47261
|
constructor(options) {
|
|
47038
|
-
this.configProvider = new Promise((
|
|
47262
|
+
this.configProvider = new Promise((resolve17, reject) => {
|
|
47039
47263
|
if (typeof options === "function") {
|
|
47040
47264
|
options().then((_options) => {
|
|
47041
|
-
|
|
47265
|
+
resolve17(this.resolveDefaultConfig(_options));
|
|
47042
47266
|
}).catch(reject);
|
|
47043
47267
|
} else {
|
|
47044
|
-
|
|
47268
|
+
resolve17(this.resolveDefaultConfig(options));
|
|
47045
47269
|
}
|
|
47046
47270
|
});
|
|
47047
47271
|
}
|
|
@@ -47072,7 +47296,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
47072
47296
|
timing.clearTimeout(socketTimeoutId);
|
|
47073
47297
|
timing.clearTimeout(keepAliveTimeoutId);
|
|
47074
47298
|
};
|
|
47075
|
-
const
|
|
47299
|
+
const resolve17 = async (arg) => {
|
|
47076
47300
|
await writeRequestBodyPromise;
|
|
47077
47301
|
clearTimeouts();
|
|
47078
47302
|
_resolve(arg);
|
|
@@ -47106,12 +47330,12 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
47106
47330
|
const password = request.password ?? "";
|
|
47107
47331
|
auth = `${username}:${password}`;
|
|
47108
47332
|
}
|
|
47109
|
-
let
|
|
47333
|
+
let path3 = request.path;
|
|
47110
47334
|
if (queryString) {
|
|
47111
|
-
|
|
47335
|
+
path3 += `?${queryString}`;
|
|
47112
47336
|
}
|
|
47113
47337
|
if (request.fragment) {
|
|
47114
|
-
|
|
47338
|
+
path3 += `#${request.fragment}`;
|
|
47115
47339
|
}
|
|
47116
47340
|
let hostname = request.hostname ?? "";
|
|
47117
47341
|
if (hostname[0] === "[" && hostname.endsWith("]")) {
|
|
@@ -47123,7 +47347,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
47123
47347
|
headers: request.headers,
|
|
47124
47348
|
host: hostname,
|
|
47125
47349
|
method: request.method,
|
|
47126
|
-
path:
|
|
47350
|
+
path: path3,
|
|
47127
47351
|
port: request.port,
|
|
47128
47352
|
agent,
|
|
47129
47353
|
auth
|
|
@@ -47136,7 +47360,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
47136
47360
|
headers: getTransformedHeaders(res.headers),
|
|
47137
47361
|
body: res
|
|
47138
47362
|
});
|
|
47139
|
-
|
|
47363
|
+
resolve17({ response: httpResponse });
|
|
47140
47364
|
});
|
|
47141
47365
|
req.on("error", (err) => {
|
|
47142
47366
|
if (NODEJS_TIMEOUT_ERROR_CODES2.includes(err.code)) {
|
|
@@ -47288,7 +47512,7 @@ var init_stream_collector2 = __esm({
|
|
|
47288
47512
|
if (isReadableStreamInstance2(stream)) {
|
|
47289
47513
|
return collectReadableStream2(stream);
|
|
47290
47514
|
}
|
|
47291
|
-
return new Promise((
|
|
47515
|
+
return new Promise((resolve17, reject) => {
|
|
47292
47516
|
const collector = new Collector2();
|
|
47293
47517
|
stream.pipe(collector);
|
|
47294
47518
|
stream.on("error", (err) => {
|
|
@@ -47298,7 +47522,7 @@ var init_stream_collector2 = __esm({
|
|
|
47298
47522
|
collector.on("error", reject);
|
|
47299
47523
|
collector.on("finish", function() {
|
|
47300
47524
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
47301
|
-
|
|
47525
|
+
resolve17(bytes);
|
|
47302
47526
|
});
|
|
47303
47527
|
});
|
|
47304
47528
|
};
|
|
@@ -47420,7 +47644,7 @@ var init_retry_wrapper = __esm({
|
|
|
47420
47644
|
try {
|
|
47421
47645
|
return await toRetry();
|
|
47422
47646
|
} catch (e6) {
|
|
47423
|
-
await new Promise((
|
|
47647
|
+
await new Promise((resolve17) => setTimeout(resolve17, delayMs));
|
|
47424
47648
|
}
|
|
47425
47649
|
}
|
|
47426
47650
|
return await toRetry();
|
|
@@ -52655,14 +52879,14 @@ var init_readableStreamHasher = __esm({
|
|
|
52655
52879
|
const hash = new hashCtor();
|
|
52656
52880
|
const hashCalculator = new HashCalculator(hash);
|
|
52657
52881
|
readableStream.pipe(hashCalculator);
|
|
52658
|
-
return new Promise((
|
|
52882
|
+
return new Promise((resolve17, reject) => {
|
|
52659
52883
|
readableStream.on("error", (err) => {
|
|
52660
52884
|
hashCalculator.end();
|
|
52661
52885
|
reject(err);
|
|
52662
52886
|
});
|
|
52663
52887
|
hashCalculator.on("error", reject);
|
|
52664
52888
|
hashCalculator.on("finish", () => {
|
|
52665
|
-
hash.digest().then(
|
|
52889
|
+
hash.digest().then(resolve17).catch(reject);
|
|
52666
52890
|
});
|
|
52667
52891
|
});
|
|
52668
52892
|
};
|
|
@@ -57184,8 +57408,8 @@ var init_dist_es22 = __esm({
|
|
|
57184
57408
|
});
|
|
57185
57409
|
|
|
57186
57410
|
// src/cli.ts
|
|
57187
|
-
import { resolve as
|
|
57188
|
-
import { readFileSync as
|
|
57411
|
+
import { resolve as resolve16, dirname as dirname18 } from "path";
|
|
57412
|
+
import { readFileSync as readFileSync25 } from "fs";
|
|
57189
57413
|
import { execSync } from "child_process";
|
|
57190
57414
|
import { parse as parse3 } from "yaml";
|
|
57191
57415
|
|
|
@@ -57453,7 +57677,7 @@ init_http();
|
|
|
57453
57677
|
import { createServer } from "http";
|
|
57454
57678
|
import { spawn as spawn2 } from "child_process";
|
|
57455
57679
|
import { WebSocketServer, WebSocket } from "ws";
|
|
57456
|
-
import { dirname as dirname17, resolve as
|
|
57680
|
+
import { dirname as dirname17, resolve as resolve15 } from "path";
|
|
57457
57681
|
|
|
57458
57682
|
// src/gui/active-db.ts
|
|
57459
57683
|
init_adapter();
|
|
@@ -57461,9 +57685,17 @@ init_members();
|
|
|
57461
57685
|
init_native_entities();
|
|
57462
57686
|
async function changeVisibleToActiveRole(db, payload) {
|
|
57463
57687
|
if (db.getDialect() !== "postgres") return true;
|
|
57464
|
-
if (payload.op === "delete" || payload.op === "DELETE") return true;
|
|
57465
57688
|
if (!payload.table_name || !payload.pk) return false;
|
|
57466
57689
|
try {
|
|
57690
|
+
if (isDeleteOp(payload.op)) {
|
|
57691
|
+
if (payload.del_owner_role == null) return false;
|
|
57692
|
+
const row2 = await getAsyncOrSync(
|
|
57693
|
+
db.adapter,
|
|
57694
|
+
`SELECT lattice_delete_visible(?, ?, ?::text[]) AS v`,
|
|
57695
|
+
[payload.del_owner_role, payload.del_visibility ?? null, payload.del_grantees ?? []]
|
|
57696
|
+
);
|
|
57697
|
+
return row2?.v === true || row2?.v === "t" || row2?.v === 1;
|
|
57698
|
+
}
|
|
57467
57699
|
const row = await getAsyncOrSync(db.adapter, `SELECT lattice_row_visible(?, ?) AS v`, [
|
|
57468
57700
|
payload.table_name,
|
|
57469
57701
|
payload.pk
|
|
@@ -58082,12 +58314,12 @@ init_postgres();
|
|
|
58082
58314
|
|
|
58083
58315
|
// src/gui/realtime.ts
|
|
58084
58316
|
import { EventEmitter } from "events";
|
|
58085
|
-
import { createRequire as
|
|
58317
|
+
import { createRequire as createRequire3 } from "module";
|
|
58086
58318
|
var _pgModule = null;
|
|
58087
58319
|
function loadPg() {
|
|
58088
58320
|
if (_pgModule) return _pgModule;
|
|
58089
58321
|
const importMetaUrl = import.meta.url;
|
|
58090
|
-
const requireFromHere = importMetaUrl ?
|
|
58322
|
+
const requireFromHere = importMetaUrl ? createRequire3(importMetaUrl) : (
|
|
58091
58323
|
// CJS fallback — Node provides `require` on every CJS module scope.
|
|
58092
58324
|
__require
|
|
58093
58325
|
);
|
|
@@ -58313,9 +58545,9 @@ var RealtimeBroker = class {
|
|
|
58313
58545
|
() => "ended"
|
|
58314
58546
|
// a graceful-close error is still "closed enough"
|
|
58315
58547
|
);
|
|
58316
|
-
const timedOut = new Promise((
|
|
58548
|
+
const timedOut = new Promise((resolve17) => {
|
|
58317
58549
|
timer = setTimeout(() => {
|
|
58318
|
-
|
|
58550
|
+
resolve17("timeout");
|
|
58319
58551
|
}, this.stopEndTimeoutMs);
|
|
58320
58552
|
timer.unref?.();
|
|
58321
58553
|
});
|
|
@@ -58360,7 +58592,10 @@ function parsePayload(raw) {
|
|
|
58360
58592
|
pk: typeof obj2.pk === "string" ? obj2.pk : null,
|
|
58361
58593
|
op: obj2.op,
|
|
58362
58594
|
owner_role: typeof obj2.owner_role === "string" ? obj2.owner_role : null,
|
|
58363
|
-
created_at: typeof obj2.created_at === "string" ? obj2.created_at : ""
|
|
58595
|
+
created_at: typeof obj2.created_at === "string" ? obj2.created_at : "",
|
|
58596
|
+
del_owner_role: typeof obj2.del_owner_role === "string" ? obj2.del_owner_role : null,
|
|
58597
|
+
del_visibility: typeof obj2.del_visibility === "string" ? obj2.del_visibility : null,
|
|
58598
|
+
del_grantees: Array.isArray(obj2.del_grantees) ? obj2.del_grantees.filter((g6) => typeof g6 === "string") : null
|
|
58364
58599
|
};
|
|
58365
58600
|
} catch {
|
|
58366
58601
|
return null;
|
|
@@ -59341,9 +59576,9 @@ function startBackgroundRender(active) {
|
|
|
59341
59576
|
}
|
|
59342
59577
|
function settleWithin(p3, ms) {
|
|
59343
59578
|
let timer;
|
|
59344
|
-
const timeout = new Promise((
|
|
59579
|
+
const timeout = new Promise((resolve17) => {
|
|
59345
59580
|
timer = setTimeout(() => {
|
|
59346
|
-
|
|
59581
|
+
resolve17("timeout");
|
|
59347
59582
|
}, ms);
|
|
59348
59583
|
timer.unref?.();
|
|
59349
59584
|
});
|
|
@@ -59398,9 +59633,9 @@ var SWITCH_OPEN_TIMEOUT_MS = 2e4;
|
|
|
59398
59633
|
async function openWithinTimeout(open, timeoutMs = SWITCH_OPEN_TIMEOUT_MS, dispose = disposeActive) {
|
|
59399
59634
|
const opening = open();
|
|
59400
59635
|
let timer;
|
|
59401
|
-
const timedOut = new Promise((
|
|
59636
|
+
const timedOut = new Promise((resolve17) => {
|
|
59402
59637
|
timer = setTimeout(() => {
|
|
59403
|
-
|
|
59638
|
+
resolve17("timeout");
|
|
59404
59639
|
}, timeoutMs);
|
|
59405
59640
|
timer.unref?.();
|
|
59406
59641
|
});
|
|
@@ -59458,7 +59693,7 @@ async function applySchemaConfig(active, entry, direction, autoRender) {
|
|
|
59458
59693
|
const doc = loadConfigDoc(active.configPath);
|
|
59459
59694
|
const inv = direction === "inverse";
|
|
59460
59695
|
const ddl = [];
|
|
59461
|
-
const has = (
|
|
59696
|
+
const has = (path3) => doc.getIn(path3) !== void 0;
|
|
59462
59697
|
const reAddEntity = async (name, def) => {
|
|
59463
59698
|
if (has(["entities", name])) {
|
|
59464
59699
|
throw new Error(`Cannot restore "${name}": an entity with that name already exists`);
|
|
@@ -61071,6 +61306,65 @@ var chatCss = ` /* \u2500\u2500 Chat bubbles + tool pills \u2500\u2500\u2500\
|
|
|
61071
61306
|
}
|
|
61072
61307
|
`;
|
|
61073
61308
|
|
|
61309
|
+
// src/gui/app/styles/inline-import.ts
|
|
61310
|
+
var inlineImportCss = `
|
|
61311
|
+
/* \u2500\u2500 Inline import confirm card (assistant rail) \u2500\u2500 */
|
|
61312
|
+
.cd-sub { margin: 10px 0 6px; font-size: 12px; color: var(--text-muted, #9aa3ad); }
|
|
61313
|
+
.cd-row { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; margin-top: 8px; }
|
|
61314
|
+
.cd-path {
|
|
61315
|
+
flex: 1 1 220px; min-width: 0; box-sizing: border-box; height: 34px; padding: 0 10px;
|
|
61316
|
+
border-radius: 6px; border: 1px solid #2a2f36;
|
|
61317
|
+
background: var(--panel, #0e1116); color: var(--text, #e6e8eb); font-size: 13px;
|
|
61318
|
+
}
|
|
61319
|
+
.cd-status { margin-top: 12px; font-size: 13px; line-height: 1.5; }
|
|
61320
|
+
.cd-status.ok { color: #bef264; }
|
|
61321
|
+
.cd-status.err { color: #f87171; }
|
|
61322
|
+
.cd-status a { color: var(--accent, #bef264); }
|
|
61323
|
+
.cd-btn {
|
|
61324
|
+
height: 34px; padding: 0 14px; border-radius: 6px; border: 1px solid #2a2f36;
|
|
61325
|
+
background: transparent; color: var(--text, #e6e8eb); font-size: 13px;
|
|
61326
|
+
font-weight: 600; cursor: pointer;
|
|
61327
|
+
}
|
|
61328
|
+
.cd-btn:hover { background: rgba(255, 255, 255, 0.06); }
|
|
61329
|
+
.cd-btn.cd-primary { background: #bef264; color: #0b0d10; border-color: #bef264; }
|
|
61330
|
+
.cd-btn.cd-primary:hover { filter: brightness(1.06); }
|
|
61331
|
+
.cd-import-list { margin: 10px 0 0; padding-left: 18px; font-size: 13px; line-height: 1.6; }
|
|
61332
|
+
.cd-import-list li { margin: 2px 0; }
|
|
61333
|
+
.imp-sub { margin: 16px 0 6px; font-size: 13px; color: var(--text, #e6e8eb); }
|
|
61334
|
+
.imp-modes { display: flex; flex-direction: column; gap: 8px; margin: 0 0 6px; }
|
|
61335
|
+
.imp-modes label {
|
|
61336
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
61337
|
+
padding: 8px 10px; border: 1px solid #2a2f36; border-radius: 6px; cursor: pointer;
|
|
61338
|
+
}
|
|
61339
|
+
.imp-modes label:hover { background: rgba(255, 255, 255, 0.04); }
|
|
61340
|
+
.imp-modes input { margin-top: 2px; }
|
|
61341
|
+
.imp-modes b { color: var(--text, #e6e8eb); }
|
|
61342
|
+
.imp-percol {
|
|
61343
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
61344
|
+
margin: 8px 0 0; cursor: pointer; color: var(--text-dim, #aeb6c2);
|
|
61345
|
+
}
|
|
61346
|
+
.imp-percol input { margin-top: 2px; }
|
|
61347
|
+
.imp-match { border-left: 3px solid var(--accent, #7dd3fc); font-weight: 500; }
|
|
61348
|
+
.feed-item.import-confirm .imp-confirm-body { margin-top: 4px; }
|
|
61349
|
+
|
|
61350
|
+
/* \u2500\u2500 Live import progress in the card's log \u2500\u2500 */
|
|
61351
|
+
.feed-item.import-confirm .imp-card-log,
|
|
61352
|
+
.feed-item.import-live .imp-card-log {
|
|
61353
|
+
margin-top: 4px;
|
|
61354
|
+
font: 12px/1.6 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
|
61355
|
+
max-height: 200px; overflow-y: auto; color: var(--text-muted, #9aa3ad);
|
|
61356
|
+
}
|
|
61357
|
+
.imp-card-line { white-space: pre-wrap; word-break: break-word; }
|
|
61358
|
+
.imp-card-line.imp-done { color: var(--accent, #bef264); }
|
|
61359
|
+
.imp-card-line.imp-err { color: #f87171; }
|
|
61360
|
+
.imp-card-line.imp-spin::after {
|
|
61361
|
+
content: ''; display: inline-block; width: 10px; height: 10px; margin-left: 7px;
|
|
61362
|
+
border: 2px solid currentColor; border-right-color: transparent; border-radius: 50%;
|
|
61363
|
+
vertical-align: -1px; animation: imp-spin-kf 0.7s linear infinite;
|
|
61364
|
+
}
|
|
61365
|
+
@keyframes imp-spin-kf { to { transform: rotate(360deg); } }
|
|
61366
|
+
`;
|
|
61367
|
+
|
|
61074
61368
|
// src/gui/app/styles/index.ts
|
|
61075
61369
|
var css = [
|
|
61076
61370
|
tokensCss,
|
|
@@ -61092,7 +61386,8 @@ var css = [
|
|
|
61092
61386
|
fsWorkspaceCss,
|
|
61093
61387
|
settingsDrawerCss,
|
|
61094
61388
|
assistantRailCss,
|
|
61095
|
-
chatCss
|
|
61389
|
+
chatCss,
|
|
61390
|
+
inlineImportCss
|
|
61096
61391
|
].join("");
|
|
61097
61392
|
|
|
61098
61393
|
// src/gui/app/modules/display-config.ts
|
|
@@ -68663,6 +68958,11 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
68663
68958
|
// survivor if it was a duplicate). Multi-file drops do not navigate.
|
|
68664
68959
|
if (files.length === 1) {
|
|
68665
68960
|
uploadFile(files[0]).then(function (j) {
|
|
68961
|
+
// A structured source the server flagged as confirmable comes back with
|
|
68962
|
+
// an autoImport proposal \u2014 render the inline confirm card instead of
|
|
68963
|
+
// navigating to the file record. A silent import (autoImport.imported,
|
|
68964
|
+
// no reason) or a plain file keeps the open-the-record behavior.
|
|
68965
|
+
if (j && j.autoImport && j.autoImport.reason) { renderInlineImportCard(j.autoImport); return; }
|
|
68666
68966
|
if (j && (j.duplicateOf || j.id)) openSearchHit('files', j.duplicateOf || j.id);
|
|
68667
68967
|
});
|
|
68668
68968
|
return;
|
|
@@ -68672,7 +68972,15 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
68672
68972
|
var bar = ingestProgress(files.length);
|
|
68673
68973
|
var thunks = [];
|
|
68674
68974
|
for (var i = 0; i < files.length; i++) {
|
|
68675
|
-
(function (f) {
|
|
68975
|
+
(function (f) {
|
|
68976
|
+
thunks.push(function () {
|
|
68977
|
+
return uploadFile(f).then(function (j) {
|
|
68978
|
+
// A structured source within a batch still gets its own inline
|
|
68979
|
+
// confirm card (the batch as a whole does not navigate).
|
|
68980
|
+
if (j && j.autoImport && j.autoImport.reason) renderInlineImportCard(j.autoImport);
|
|
68981
|
+
});
|
|
68982
|
+
});
|
|
68983
|
+
})(files[i]);
|
|
68676
68984
|
}
|
|
68677
68985
|
runIngestBatch(thunks, INGEST_MAX_CONCURRENCY, bar.update).then(bar.done);
|
|
68678
68986
|
}
|
|
@@ -68828,6 +69136,237 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
68828
69136
|
})();
|
|
68829
69137
|
`;
|
|
68830
69138
|
|
|
69139
|
+
// src/gui/app/modules/inline-import.ts
|
|
69140
|
+
var inlineImportJs = `
|
|
69141
|
+
// \u2500\u2500 Inline structured-source import (confirm card in the assistant rail) \u2500\u2500
|
|
69142
|
+
function iiRailFeed() { return document.getElementById('rail-feed'); }
|
|
69143
|
+
function iiRailEmptyGone() {
|
|
69144
|
+
var e = document.getElementById('rail-empty');
|
|
69145
|
+
if (e) e.parentNode && e.parentNode.removeChild(e);
|
|
69146
|
+
}
|
|
69147
|
+
|
|
69148
|
+
// Read a newline-delimited-JSON response body, invoking onEvent(obj) per line.
|
|
69149
|
+
// Self-contained on purpose \u2014 this segment must not depend on any other.
|
|
69150
|
+
function iiStreamNdjson(url, payload, onEvent) {
|
|
69151
|
+
fetch(url, {
|
|
69152
|
+
method: 'POST',
|
|
69153
|
+
headers: { 'content-type': 'application/json' },
|
|
69154
|
+
body: JSON.stringify(payload),
|
|
69155
|
+
}).then(function (res) {
|
|
69156
|
+
if (!res.body || !res.body.getReader) {
|
|
69157
|
+
return res.text().then(function (t) {
|
|
69158
|
+
t.split('\\n').forEach(function (line) {
|
|
69159
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
69160
|
+
});
|
|
69161
|
+
});
|
|
69162
|
+
}
|
|
69163
|
+
var reader = res.body.getReader();
|
|
69164
|
+
var dec = new TextDecoder();
|
|
69165
|
+
var buf = '';
|
|
69166
|
+
function pump() {
|
|
69167
|
+
return reader.read().then(function (chunk) {
|
|
69168
|
+
if (chunk.done) {
|
|
69169
|
+
if (buf.trim()) { try { onEvent(JSON.parse(buf)); } catch (e) { /* skip */ } }
|
|
69170
|
+
return;
|
|
69171
|
+
}
|
|
69172
|
+
buf += dec.decode(chunk.value, { stream: true });
|
|
69173
|
+
var idx;
|
|
69174
|
+
while ((idx = buf.indexOf('\\n')) >= 0) {
|
|
69175
|
+
var line = buf.slice(0, idx);
|
|
69176
|
+
buf = buf.slice(idx + 1);
|
|
69177
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
69178
|
+
}
|
|
69179
|
+
return pump();
|
|
69180
|
+
});
|
|
69181
|
+
}
|
|
69182
|
+
return pump();
|
|
69183
|
+
}).catch(function (err) {
|
|
69184
|
+
onEvent({ phase: 'error', message: err && err.message ? err.message : 'Request failed' });
|
|
69185
|
+
});
|
|
69186
|
+
}
|
|
69187
|
+
|
|
69188
|
+
// Render the confirm card for a structured drop the server flagged as
|
|
69189
|
+
// needing confirmation. autoImport is the upload response's proposal:
|
|
69190
|
+
// { reason, fileId, plan:{entities,dimensions,linkages}, views, asOf,
|
|
69191
|
+
// asOfCandidates, asOfColumns, schemaMatch, matchedCount, totalEntities }.
|
|
69192
|
+
function renderInlineImportCard(autoImport) {
|
|
69193
|
+
if (!autoImport || !autoImport.fileId) return;
|
|
69194
|
+
var plan = autoImport.plan || {};
|
|
69195
|
+
var ents = plan.entities || [];
|
|
69196
|
+
var dims = plan.dimensions || [];
|
|
69197
|
+
var links = plan.linkages || [];
|
|
69198
|
+
var views = autoImport.views || [];
|
|
69199
|
+
var candidates = autoImport.asOfCandidates || [];
|
|
69200
|
+
var asOfColumns = autoImport.asOfColumns || [];
|
|
69201
|
+
var schemaMatch = autoImport.schemaMatch || {};
|
|
69202
|
+
var headerText = autoImport.reason === 'needs-confirm'
|
|
69203
|
+
? 'Add a dated snapshot'
|
|
69204
|
+
: 'Import as a new dataset';
|
|
69205
|
+
|
|
69206
|
+
iiRailEmptyGone();
|
|
69207
|
+
var feedEl = iiRailFeed();
|
|
69208
|
+
var card = document.createElement('div');
|
|
69209
|
+
card.className = 'feed-item import-confirm';
|
|
69210
|
+
var icon = document.createElement('div');
|
|
69211
|
+
icon.className = 'feed-icon';
|
|
69212
|
+
icon.textContent = '\u2913';
|
|
69213
|
+
var bodyEl = document.createElement('div');
|
|
69214
|
+
bodyEl.className = 'feed-body';
|
|
69215
|
+
var title = document.createElement('div');
|
|
69216
|
+
title.className = 'feed-summary';
|
|
69217
|
+
title.textContent = headerText;
|
|
69218
|
+
bodyEl.appendChild(title);
|
|
69219
|
+
|
|
69220
|
+
var parts = [];
|
|
69221
|
+
if (schemaMatch.isKnownDocument) {
|
|
69222
|
+
parts.push('<div class="cd-status ok imp-match">Recognized as a new period of an existing document — ' +
|
|
69223
|
+
schemaMatch.matchedCount + ' of ' + schemaMatch.totalEntities +
|
|
69224
|
+
' tables match what you already imported. It will be added as a dated snapshot.</div>');
|
|
69225
|
+
}
|
|
69226
|
+
parts.push('<div class="cd-status ok">Found ' + ents.length + ' entities, ' + dims.length +
|
|
69227
|
+
' dimensions, ' + links.length + ' links' +
|
|
69228
|
+
(views.length ? ', ' + views.length + ' reconstructed views (no duplicated rows)' : '') +
|
|
69229
|
+
'.</div><ul class="cd-import-list">');
|
|
69230
|
+
ents.forEach(function (e) {
|
|
69231
|
+
parts.push('<li><b>' + escapeHtml(e.name) + '</b> — ' + e.rowCount + ' rows, ' +
|
|
69232
|
+
(e.columns ? e.columns.length : 0) + ' cols · ' +
|
|
69233
|
+
(e.naturalKey ? 'key ' + escapeHtml(e.naturalKey) : 'keyless') + '</li>');
|
|
69234
|
+
});
|
|
69235
|
+
dims.forEach(function (d) {
|
|
69236
|
+
parts.push('<li><b>' + escapeHtml(d.name) + '</b> (dimension) — ' + d.distinctValues + ' values</li>');
|
|
69237
|
+
});
|
|
69238
|
+
views.forEach(function (v) {
|
|
69239
|
+
parts.push('<li><b>' + escapeHtml(v.name) + '</b> (view of ' + escapeHtml(v.master) + ' where ' +
|
|
69240
|
+
escapeHtml(v.filterColumn) + ' = ' + escapeHtml(String(v.filterValue)) + ') — ' +
|
|
69241
|
+
v.matchedRows + ' rows, not duplicated</li>');
|
|
69242
|
+
});
|
|
69243
|
+
parts.push('</ul>');
|
|
69244
|
+
|
|
69245
|
+
parts.push('<h4 class="imp-sub">As of date</h4>');
|
|
69246
|
+
var best = candidates[0];
|
|
69247
|
+
parts.push('<p class="cd-sub">' +
|
|
69248
|
+
(best ? 'Detected from ' + escapeHtml(best.evidence) + ' — edit if wrong.'
|
|
69249
|
+
: 'No date found in the file or its name — set the snapshot date, or leave blank to import undated.') +
|
|
69250
|
+
' A newer file is kept as a separate dated snapshot beside the prior one.</p>');
|
|
69251
|
+
parts.push('<div class="cd-row"><input class="cd-path" id="ii-asof" type="date" value="' + escapeHtml(autoImport.asOf || '') + '" aria-label="As of date" /></div>');
|
|
69252
|
+
if (candidates.length > 1) {
|
|
69253
|
+
parts.push('<div class="cd-sub">Other candidates: ' + candidates.slice(1, 5).map(function (c) {
|
|
69254
|
+
return '<a href="#" class="ii-asof-alt" data-date="' + escapeHtml(c.date) + '" title="' + escapeHtml(c.evidence) + '">' + escapeHtml(c.date) + '</a>';
|
|
69255
|
+
}).join(', ') + '</div>');
|
|
69256
|
+
}
|
|
69257
|
+
if (asOfColumns.length) {
|
|
69258
|
+
var colOpts = asOfColumns.slice(0, 6).map(function (c) {
|
|
69259
|
+
return '<option value="' + escapeHtml(c.column) + '" title="' + escapeHtml(c.evidence) + '">' +
|
|
69260
|
+
escapeHtml(c.column) + ' (' + escapeHtml(c.entity) + ', ' + c.distinctDates +
|
|
69261
|
+
' date' + (c.distinctDates === 1 ? '' : 's') + ')</option>';
|
|
69262
|
+
}).join('');
|
|
69263
|
+
parts.push('<label class="imp-percol"><input type="checkbox" id="ii-asof-percol"> ' +
|
|
69264
|
+
'<span>Date varies per row — use a date column instead (one file, many periods)</span></label>');
|
|
69265
|
+
parts.push('<div class="cd-row" id="ii-asof-col-row" style="display:none"><select class="cd-path" id="ii-asof-col">' + colOpts + '</select></div>');
|
|
69266
|
+
}
|
|
69267
|
+
|
|
69268
|
+
parts.push('<h4 class="imp-sub">What should Lattice bring in?</h4>');
|
|
69269
|
+
parts.push('<div class="imp-modes">' +
|
|
69270
|
+
'<label><input type="radio" name="ii-mode" value="both" checked> <span><b>Data model + contents</b> \u2014 the schema, the taxonomy, and all the rows.</span></label>' +
|
|
69271
|
+
'<label><input type="radio" name="ii-mode" value="schema"> <span><b>Data model / schema only</b> \u2014 tables, dimension values, and views. No rows.</span></label>' +
|
|
69272
|
+
'<label><input type="radio" name="ii-mode" value="contents"> <span><b>Contents only</b> \u2014 the rows and their links, into tables that already exist.</span></label>' +
|
|
69273
|
+
'</div>');
|
|
69274
|
+
parts.push('<div class="cd-row"><button class="cd-btn cd-primary" id="ii-apply" type="button">Import into Lattice</button></div>');
|
|
69275
|
+
parts.push('<div class="imp-card-log" id="ii-log"></div>');
|
|
69276
|
+
|
|
69277
|
+
var content = document.createElement('div');
|
|
69278
|
+
content.className = 'imp-confirm-body';
|
|
69279
|
+
content.innerHTML = parts.join('');
|
|
69280
|
+
bodyEl.appendChild(content);
|
|
69281
|
+
card.appendChild(icon);
|
|
69282
|
+
card.appendChild(bodyEl);
|
|
69283
|
+
if (feedEl) { feedEl.appendChild(card); feedEl.scrollTop = feedEl.scrollHeight; }
|
|
69284
|
+
|
|
69285
|
+
content.querySelectorAll('.ii-asof-alt').forEach(function (a) {
|
|
69286
|
+
a.addEventListener('click', function (e) {
|
|
69287
|
+
e.preventDefault();
|
|
69288
|
+
var input = document.getElementById('ii-asof');
|
|
69289
|
+
if (input) input.value = a.getAttribute('data-date') || '';
|
|
69290
|
+
});
|
|
69291
|
+
});
|
|
69292
|
+
var perCol = document.getElementById('ii-asof-percol');
|
|
69293
|
+
if (perCol) perCol.addEventListener('change', function () {
|
|
69294
|
+
var row = document.getElementById('ii-asof-col-row');
|
|
69295
|
+
var dateEl = document.getElementById('ii-asof');
|
|
69296
|
+
if (row) row.style.display = perCol.checked ? '' : 'none';
|
|
69297
|
+
if (dateEl) dateEl.disabled = perCol.checked;
|
|
69298
|
+
});
|
|
69299
|
+
|
|
69300
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
69301
|
+
if (applyBtn) applyBtn.addEventListener('click', function () {
|
|
69302
|
+
runInlineImport(autoImport.fileId, title, content);
|
|
69303
|
+
});
|
|
69304
|
+
}
|
|
69305
|
+
|
|
69306
|
+
// POST the confirmed proposal to /api/import/apply and stream the pipeline
|
|
69307
|
+
// live into the card's log. On 'done' show a success summary + refresh the
|
|
69308
|
+
// Objects nav in place; on 'error' show the message.
|
|
69309
|
+
function runInlineImport(fileId, title, content) {
|
|
69310
|
+
var sel = content.querySelector('input[name="ii-mode"]:checked');
|
|
69311
|
+
var mode = sel ? sel.value : 'both';
|
|
69312
|
+
var asofEl = document.getElementById('ii-asof');
|
|
69313
|
+
var asOf = asofEl ? asofEl.value : '';
|
|
69314
|
+
var perColEl = document.getElementById('ii-asof-percol');
|
|
69315
|
+
var colSel = document.getElementById('ii-asof-col');
|
|
69316
|
+
var asOfColumn = (perColEl && perColEl.checked && colSel) ? colSel.value : '';
|
|
69317
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
69318
|
+
if (applyBtn) applyBtn.disabled = true;
|
|
69319
|
+
|
|
69320
|
+
var feedEl = iiRailFeed();
|
|
69321
|
+
var log = document.getElementById('ii-log');
|
|
69322
|
+
function addLine(text, cls) {
|
|
69323
|
+
if (!log) return null;
|
|
69324
|
+
var d = document.createElement('div');
|
|
69325
|
+
d.className = 'imp-card-line' + (cls ? ' ' + cls : '');
|
|
69326
|
+
d.textContent = text;
|
|
69327
|
+
log.appendChild(d);
|
|
69328
|
+
while (log.childNodes.length > 60) log.removeChild(log.firstChild);
|
|
69329
|
+
log.scrollTop = log.scrollHeight;
|
|
69330
|
+
if (feedEl) feedEl.scrollTop = feedEl.scrollHeight;
|
|
69331
|
+
return d;
|
|
69332
|
+
}
|
|
69333
|
+
title.textContent = 'Importing your data\u2026';
|
|
69334
|
+
addLine('Starting\u2026');
|
|
69335
|
+
|
|
69336
|
+
iiStreamNdjson('/api/import/apply', { fileId: fileId, mode: mode, asOf: asOf, asOfColumn: asOfColumn }, function (evt) {
|
|
69337
|
+
if (!evt) return;
|
|
69338
|
+
if (evt.phase === 'done') {
|
|
69339
|
+
var r = evt.result || {};
|
|
69340
|
+
var rbt = r.rowsByTable || {};
|
|
69341
|
+
var names = Object.keys(rbt);
|
|
69342
|
+
var total = 0;
|
|
69343
|
+
names.forEach(function (n) { total += (rbt[n] || 0); });
|
|
69344
|
+
title.textContent = 'Imported ' + names.length + ' tables' + (mode === 'schema' ? '' : ', ' + total + ' rows');
|
|
69345
|
+
var upd = addLine('Updating your objects\u2026', 'imp-spin');
|
|
69346
|
+
refreshEntities().then(function () {
|
|
69347
|
+
renderSidebar();
|
|
69348
|
+
renderRoute();
|
|
69349
|
+
var count = (state.entities && state.entities.tables) ? state.entities.tables.length : names.length;
|
|
69350
|
+
if (upd) {
|
|
69351
|
+
upd.className = 'imp-card-line imp-done';
|
|
69352
|
+
upd.textContent = '\u2713 Done \u2014 ' + count + ' objects in your workspace';
|
|
69353
|
+
}
|
|
69354
|
+
}).catch(function () {
|
|
69355
|
+
if (upd) {
|
|
69356
|
+
upd.className = 'imp-card-line imp-err';
|
|
69357
|
+
upd.textContent = 'Imported, but refreshing the view failed \u2014 reload to see your objects.';
|
|
69358
|
+
}
|
|
69359
|
+
});
|
|
69360
|
+
} else if (evt.phase === 'error') {
|
|
69361
|
+
title.textContent = 'Import failed';
|
|
69362
|
+
addLine('Error: ' + (evt.message || 'import failed'), 'imp-err');
|
|
69363
|
+
} else if (evt.message) {
|
|
69364
|
+
addLine(evt.message);
|
|
69365
|
+
}
|
|
69366
|
+
});
|
|
69367
|
+
}
|
|
69368
|
+
`;
|
|
69369
|
+
|
|
68831
69370
|
// src/gui/app/modules/index.ts
|
|
68832
69371
|
var appJs = [
|
|
68833
69372
|
displayConfigJs,
|
|
@@ -68856,7 +69395,8 @@ var appJs = [
|
|
|
68856
69395
|
dataModelJs,
|
|
68857
69396
|
latticeTeamsJs,
|
|
68858
69397
|
onboardingJs,
|
|
68859
|
-
createDatabaseWizardJs
|
|
69398
|
+
createDatabaseWizardJs,
|
|
69399
|
+
inlineImportJs
|
|
68860
69400
|
].join("");
|
|
68861
69401
|
|
|
68862
69402
|
// src/gui/app/analytics.ts
|
|
@@ -69696,9 +70236,9 @@ function rewriteDbLine(configPath, newValue) {
|
|
|
69696
70236
|
function parseSaveBody(body) {
|
|
69697
70237
|
const type = body.type;
|
|
69698
70238
|
if (type === "sqlite") {
|
|
69699
|
-
const
|
|
69700
|
-
if (!
|
|
69701
|
-
return { type: "sqlite", path:
|
|
70239
|
+
const path3 = typeof body.path === "string" && body.path.trim() ? body.path.trim() : "";
|
|
70240
|
+
if (!path3) return null;
|
|
70241
|
+
return { type: "sqlite", path: path3 };
|
|
69702
70242
|
}
|
|
69703
70243
|
if (type === "postgres") {
|
|
69704
70244
|
const label = typeof body.label === "string" && body.label.trim() ? body.label.trim() : "";
|
|
@@ -71432,16 +71972,16 @@ init_extract();
|
|
|
71432
71972
|
import { statSync as statSync8 } from "fs";
|
|
71433
71973
|
import { writeFile as writeFile2, rm } from "fs/promises";
|
|
71434
71974
|
import { tmpdir as tmpdir2 } from "os";
|
|
71435
|
-
import { basename as
|
|
71975
|
+
import { basename as basename11, extname as extname2, resolve as resolve11, join as join28 } from "path";
|
|
71436
71976
|
|
|
71437
71977
|
// src/ai/vision.ts
|
|
71438
71978
|
init_llm_client();
|
|
71439
|
-
import { createRequire as
|
|
71979
|
+
import { createRequire as createRequire7 } from "module";
|
|
71440
71980
|
import { readFile as readFile8 } from "fs/promises";
|
|
71441
71981
|
var DEFAULT_PROMPT = "Describe this image for a knowledge base in 2-4 factual sentences: what it shows, any visible text, and notable details. No preamble.";
|
|
71442
71982
|
var MAX_DIM = 1568;
|
|
71443
|
-
async function describeImage(auth,
|
|
71444
|
-
const data = (await normalizeImage(
|
|
71983
|
+
async function describeImage(auth, path3, opts = {}) {
|
|
71984
|
+
const data = (await normalizeImage(path3, opts.maxBytes ?? 14e5)).toString("base64");
|
|
71445
71985
|
const sender = opts.sender ?? defaultSender(auth);
|
|
71446
71986
|
const text = await sender({
|
|
71447
71987
|
media_type: "image/jpeg",
|
|
@@ -71452,8 +71992,8 @@ async function describeImage(auth, path2, opts = {}) {
|
|
|
71452
71992
|
return text.trim();
|
|
71453
71993
|
}
|
|
71454
71994
|
var DEFAULT_PDF_PROMPT = "Read this document for a knowledge base. First transcribe its readable text, then add a 2-4 sentence factual summary of what it is and its key details. It may be a scanned/image-only PDF \u2014 read the text from the page images. No preamble.";
|
|
71455
|
-
async function describePdf(auth,
|
|
71456
|
-
const buf = await readFile8(
|
|
71995
|
+
async function describePdf(auth, path3, opts = {}) {
|
|
71996
|
+
const buf = await readFile8(path3);
|
|
71457
71997
|
const maxBytes = opts.maxBytes ?? 3e7;
|
|
71458
71998
|
if (buf.length > maxBytes) {
|
|
71459
71999
|
throw new Error(
|
|
@@ -71468,19 +72008,19 @@ async function describePdf(auth, path2, opts = {}) {
|
|
|
71468
72008
|
});
|
|
71469
72009
|
return text.trim();
|
|
71470
72010
|
}
|
|
71471
|
-
async function normalizeImage(
|
|
72011
|
+
async function normalizeImage(path3, maxBytes) {
|
|
71472
72012
|
const sharpMod = await import("sharp");
|
|
71473
72013
|
const sharp = sharpMod.default;
|
|
71474
72014
|
let quality = 80;
|
|
71475
|
-
let buf = await renderJpeg(sharp,
|
|
72015
|
+
let buf = await renderJpeg(sharp, path3, quality);
|
|
71476
72016
|
while (buf.length > maxBytes && quality > 35) {
|
|
71477
72017
|
quality -= 15;
|
|
71478
|
-
buf = await renderJpeg(sharp,
|
|
72018
|
+
buf = await renderJpeg(sharp, path3, quality);
|
|
71479
72019
|
}
|
|
71480
72020
|
return buf;
|
|
71481
72021
|
}
|
|
71482
|
-
function renderJpeg(sharp,
|
|
71483
|
-
return sharp(
|
|
72022
|
+
function renderJpeg(sharp, path3, quality) {
|
|
72023
|
+
return sharp(path3).rotate().resize({ width: MAX_DIM, height: MAX_DIM, fit: "inside", withoutEnlargement: true }).jpeg({ quality }).toBuffer();
|
|
71484
72024
|
}
|
|
71485
72025
|
function buildVisionAnthropicConfig(auth) {
|
|
71486
72026
|
const config = {};
|
|
@@ -71498,7 +72038,7 @@ function buildVisionAnthropicConfig(auth) {
|
|
|
71498
72038
|
function defaultSender(auth) {
|
|
71499
72039
|
return async (input) => {
|
|
71500
72040
|
const importMetaUrl = import.meta.url;
|
|
71501
|
-
const req = importMetaUrl ?
|
|
72041
|
+
const req = importMetaUrl ? createRequire7(importMetaUrl) : __require;
|
|
71502
72042
|
const sdk = req("@anthropic-ai/sdk");
|
|
71503
72043
|
const Anthropic = sdk.Anthropic ?? sdk.default;
|
|
71504
72044
|
if (!Anthropic) throw new Error("Could not resolve Anthropic from '@anthropic-ai/sdk'");
|
|
@@ -71525,7 +72065,7 @@ function defaultSender(auth) {
|
|
|
71525
72065
|
function defaultPdfSender(auth) {
|
|
71526
72066
|
return async (input) => {
|
|
71527
72067
|
const importMetaUrl = import.meta.url;
|
|
71528
|
-
const req = importMetaUrl ?
|
|
72068
|
+
const req = importMetaUrl ? createRequire7(importMetaUrl) : __require;
|
|
71529
72069
|
const sdk = req("@anthropic-ai/sdk");
|
|
71530
72070
|
const Anthropic = sdk.Anthropic ?? sdk.default;
|
|
71531
72071
|
if (!Anthropic) throw new Error("Could not resolve Anthropic from '@anthropic-ai/sdk'");
|
|
@@ -71555,13 +72095,13 @@ import { createHash as createHash10 } from "crypto";
|
|
|
71555
72095
|
import { createReadStream as createReadStream2, existsSync as existsSync25, mkdirSync as mkdirSync11, statSync as statSync7, copyFileSync as copyFileSync4 } from "fs";
|
|
71556
72096
|
import { basename as basename9, join as join27 } from "path";
|
|
71557
72097
|
async function hashFile(srcPath) {
|
|
71558
|
-
return new Promise((
|
|
72098
|
+
return new Promise((resolve17, reject) => {
|
|
71559
72099
|
const hash = createHash10("sha256");
|
|
71560
72100
|
const stream = createReadStream2(srcPath);
|
|
71561
72101
|
stream.on("data", (chunk) => hash.update(chunk));
|
|
71562
72102
|
stream.on("error", reject);
|
|
71563
72103
|
stream.on("end", () => {
|
|
71564
|
-
|
|
72104
|
+
resolve17(hash.digest("hex"));
|
|
71565
72105
|
});
|
|
71566
72106
|
});
|
|
71567
72107
|
}
|
|
@@ -71593,8 +72133,1158 @@ init_http();
|
|
|
71593
72133
|
init_enrich();
|
|
71594
72134
|
init_ingest_url();
|
|
71595
72135
|
init_file_row();
|
|
71596
|
-
import { createHash as
|
|
72136
|
+
import { createHash as createHash12 } from "crypto";
|
|
71597
72137
|
init_dedup_service();
|
|
72138
|
+
|
|
72139
|
+
// src/gui/import-auto.ts
|
|
72140
|
+
import { readFileSync as readFileSync22 } from "fs";
|
|
72141
|
+
|
|
72142
|
+
// src/import/infer.ts
|
|
72143
|
+
var SAMPLE = 300;
|
|
72144
|
+
var PREFERRED_KEYS = ["code", "id", "slug", "key", "ticker", "symbol"];
|
|
72145
|
+
var NEVER_KEY = /* @__PURE__ */ new Set([
|
|
72146
|
+
"description",
|
|
72147
|
+
"notes",
|
|
72148
|
+
"summary",
|
|
72149
|
+
"desc",
|
|
72150
|
+
"comment",
|
|
72151
|
+
"comments",
|
|
72152
|
+
"bio",
|
|
72153
|
+
"text",
|
|
72154
|
+
"body"
|
|
72155
|
+
]);
|
|
72156
|
+
var FREETEXT = /* @__PURE__ */ new Set([...NEVER_KEY, "name", "title", "company", "label"]);
|
|
72157
|
+
var DIM_MAX_DISTINCT = 64;
|
|
72158
|
+
var DIM_MAX_RATIO = 0.5;
|
|
72159
|
+
var LINK_MIN_CONFIDENCE = 0.3;
|
|
72160
|
+
function isPlainObject(v2) {
|
|
72161
|
+
return typeof v2 === "object" && v2 !== null && !Array.isArray(v2);
|
|
72162
|
+
}
|
|
72163
|
+
function sourceRecords(data, entity) {
|
|
72164
|
+
const v2 = data[entity.sourceKey];
|
|
72165
|
+
if (!Array.isArray(v2)) return [];
|
|
72166
|
+
if (entity.columnar) {
|
|
72167
|
+
const cols = data[entity.sourceKey + "Cols"];
|
|
72168
|
+
if (!Array.isArray(cols)) return [];
|
|
72169
|
+
return v2.map((row) => {
|
|
72170
|
+
const o3 = {};
|
|
72171
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
72172
|
+
return o3;
|
|
72173
|
+
});
|
|
72174
|
+
}
|
|
72175
|
+
return v2.filter(isPlainObject);
|
|
72176
|
+
}
|
|
72177
|
+
function normalizeName(key) {
|
|
72178
|
+
const s2 = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
72179
|
+
if (!s2) return "field";
|
|
72180
|
+
return /^[a-z]/.test(s2) ? s2 : "f_" + s2;
|
|
72181
|
+
}
|
|
72182
|
+
var ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
72183
|
+
var ISO_DATETIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/;
|
|
72184
|
+
function inferFieldType(values) {
|
|
72185
|
+
const present = values.filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
72186
|
+
if (present.length === 0) return "text";
|
|
72187
|
+
if (present.every((v2) => typeof v2 === "number")) {
|
|
72188
|
+
return present.every((v2) => Number.isInteger(v2)) ? "integer" : "real";
|
|
72189
|
+
}
|
|
72190
|
+
if (present.every((v2) => typeof v2 === "boolean")) return "boolean";
|
|
72191
|
+
if (present.every((v2) => typeof v2 === "string")) {
|
|
72192
|
+
if (present.every((v2) => ISO_DATE.test(v2))) return "date";
|
|
72193
|
+
if (present.every((v2) => ISO_DATETIME.test(v2))) return "datetime";
|
|
72194
|
+
}
|
|
72195
|
+
return "text";
|
|
72196
|
+
}
|
|
72197
|
+
function norm2(v2) {
|
|
72198
|
+
return String(v2).trim().toLowerCase();
|
|
72199
|
+
}
|
|
72200
|
+
function isNumericValue(v2) {
|
|
72201
|
+
if (typeof v2 === "number") return Number.isFinite(v2);
|
|
72202
|
+
if (typeof v2 !== "string") return false;
|
|
72203
|
+
const s2 = v2.replace(/[\s,$%()]/g, "");
|
|
72204
|
+
return s2 !== "" && Number.isFinite(Number(s2));
|
|
72205
|
+
}
|
|
72206
|
+
function profileColumns(records) {
|
|
72207
|
+
const keys = /* @__PURE__ */ new Set();
|
|
72208
|
+
for (const r6 of records.slice(0, SAMPLE)) for (const k6 of Object.keys(r6)) keys.add(k6);
|
|
72209
|
+
const out = /* @__PURE__ */ new Map();
|
|
72210
|
+
for (const key of keys) {
|
|
72211
|
+
let isArray = false;
|
|
72212
|
+
const sample = [];
|
|
72213
|
+
const valueSet = /* @__PURE__ */ new Set();
|
|
72214
|
+
const distinctSet = /* @__PURE__ */ new Set();
|
|
72215
|
+
let nonNull = 0;
|
|
72216
|
+
let numeric = 0;
|
|
72217
|
+
for (const r6 of records) {
|
|
72218
|
+
const v2 = r6[key];
|
|
72219
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
72220
|
+
nonNull++;
|
|
72221
|
+
if (Array.isArray(v2)) {
|
|
72222
|
+
isArray = true;
|
|
72223
|
+
for (const e6 of v2) {
|
|
72224
|
+
if (e6 !== null && e6 !== void 0 && e6 !== "") {
|
|
72225
|
+
valueSet.add(norm2(e6));
|
|
72226
|
+
distinctSet.add(norm2(e6));
|
|
72227
|
+
}
|
|
72228
|
+
}
|
|
72229
|
+
} else {
|
|
72230
|
+
if (sample.length < SAMPLE) sample.push(v2);
|
|
72231
|
+
if (typeof v2 === "string") valueSet.add(norm2(v2));
|
|
72232
|
+
distinctSet.add(norm2(v2));
|
|
72233
|
+
if (isNumericValue(v2)) numeric++;
|
|
72234
|
+
}
|
|
72235
|
+
}
|
|
72236
|
+
out.set(key, {
|
|
72237
|
+
sourceKey: key,
|
|
72238
|
+
isArray,
|
|
72239
|
+
type: isArray ? "text" : inferFieldType(sample),
|
|
72240
|
+
// Cardinality counts ALL distinct values (numbers + strings). Counting only
|
|
72241
|
+
// string values let a mostly-numeric column with a few text sentinels (e.g.
|
|
72242
|
+
// a "TEV/EBITDA" of numbers + "NM") look low-cardinality and slip in as a
|
|
72243
|
+
// junk dimension.
|
|
72244
|
+
distinct: distinctSet.size,
|
|
72245
|
+
valueSet,
|
|
72246
|
+
numericFraction: nonNull > 0 ? numeric / nonNull : 0
|
|
72247
|
+
});
|
|
72248
|
+
}
|
|
72249
|
+
return out;
|
|
72250
|
+
}
|
|
72251
|
+
function pickNaturalKey(records, profiles) {
|
|
72252
|
+
const n3 = records.length;
|
|
72253
|
+
const isUnique = (key) => {
|
|
72254
|
+
const seen = /* @__PURE__ */ new Set();
|
|
72255
|
+
for (const r6 of records) {
|
|
72256
|
+
const v2 = r6[key];
|
|
72257
|
+
if (v2 === null || v2 === void 0 || v2 === "") return false;
|
|
72258
|
+
const k6 = norm2(v2);
|
|
72259
|
+
if (seen.has(k6)) return false;
|
|
72260
|
+
seen.add(k6);
|
|
72261
|
+
}
|
|
72262
|
+
return seen.size === n3;
|
|
72263
|
+
};
|
|
72264
|
+
for (const pref of PREFERRED_KEYS) {
|
|
72265
|
+
for (const [key, p3] of profiles) {
|
|
72266
|
+
if (p3.isArray) continue;
|
|
72267
|
+
if (normalizeName(key) === pref && isUnique(key)) return key;
|
|
72268
|
+
}
|
|
72269
|
+
}
|
|
72270
|
+
for (const [key, p3] of profiles) {
|
|
72271
|
+
if (p3.isArray) continue;
|
|
72272
|
+
if (NEVER_KEY.has(normalizeName(key))) continue;
|
|
72273
|
+
if ((p3.type === "text" || p3.type === "integer") && isUnique(key)) return key;
|
|
72274
|
+
}
|
|
72275
|
+
return null;
|
|
72276
|
+
}
|
|
72277
|
+
function inferSchema(data, opts = {}) {
|
|
72278
|
+
const skipped = [];
|
|
72279
|
+
const consumedColsKeys = /* @__PURE__ */ new Set();
|
|
72280
|
+
for (const key of Object.keys(data)) {
|
|
72281
|
+
const v2 = data[key];
|
|
72282
|
+
const cols = data[key + "Cols"];
|
|
72283
|
+
if (Array.isArray(v2) && v2.length > 0 && Array.isArray(v2[0]) && Array.isArray(cols) && cols.every((c6) => typeof c6 === "string")) {
|
|
72284
|
+
consumedColsKeys.add(key + "Cols");
|
|
72285
|
+
}
|
|
72286
|
+
}
|
|
72287
|
+
const sources = [];
|
|
72288
|
+
for (const key of Object.keys(data)) {
|
|
72289
|
+
if (consumedColsKeys.has(key)) continue;
|
|
72290
|
+
const v2 = data[key];
|
|
72291
|
+
if (!Array.isArray(v2) || v2.length === 0) {
|
|
72292
|
+
skipped.push({
|
|
72293
|
+
key,
|
|
72294
|
+
reason: isPlainObject(v2) ? "object (derived/rollup)" : "scalar/empty (meta or derived)"
|
|
72295
|
+
});
|
|
72296
|
+
continue;
|
|
72297
|
+
}
|
|
72298
|
+
let records;
|
|
72299
|
+
let columnar = false;
|
|
72300
|
+
if (isPlainObject(v2[0])) {
|
|
72301
|
+
records = v2.filter(isPlainObject);
|
|
72302
|
+
} else if (Array.isArray(v2[0]) && Array.isArray(data[key + "Cols"])) {
|
|
72303
|
+
const cols = data[key + "Cols"];
|
|
72304
|
+
records = v2.map((row) => {
|
|
72305
|
+
const o3 = {};
|
|
72306
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
72307
|
+
return o3;
|
|
72308
|
+
});
|
|
72309
|
+
columnar = true;
|
|
72310
|
+
} else {
|
|
72311
|
+
skipped.push({ key, reason: "array of scalars (not a record set)" });
|
|
72312
|
+
continue;
|
|
72313
|
+
}
|
|
72314
|
+
const name = opts.rename?.[key] ?? normalizeName(key);
|
|
72315
|
+
const profiles = profileColumns(records);
|
|
72316
|
+
sources.push({
|
|
72317
|
+
name,
|
|
72318
|
+
sourceKey: key,
|
|
72319
|
+
records,
|
|
72320
|
+
columnar,
|
|
72321
|
+
profiles,
|
|
72322
|
+
naturalKey: pickNaturalKey(records, profiles)
|
|
72323
|
+
});
|
|
72324
|
+
}
|
|
72325
|
+
const linkages = [];
|
|
72326
|
+
const consumedFields = /* @__PURE__ */ new Map();
|
|
72327
|
+
const linkedTargets = /* @__PURE__ */ new Map();
|
|
72328
|
+
const consume = (e6, f6) => {
|
|
72329
|
+
let set = consumedFields.get(e6);
|
|
72330
|
+
if (!set) {
|
|
72331
|
+
set = /* @__PURE__ */ new Set();
|
|
72332
|
+
consumedFields.set(e6, set);
|
|
72333
|
+
}
|
|
72334
|
+
set.add(f6);
|
|
72335
|
+
};
|
|
72336
|
+
const markTarget = (e6, t8) => {
|
|
72337
|
+
let set = linkedTargets.get(e6);
|
|
72338
|
+
if (!set) {
|
|
72339
|
+
set = /* @__PURE__ */ new Set();
|
|
72340
|
+
linkedTargets.set(e6, set);
|
|
72341
|
+
}
|
|
72342
|
+
set.add(t8);
|
|
72343
|
+
};
|
|
72344
|
+
function bestTarget(self, values) {
|
|
72345
|
+
if (values.size === 0) return null;
|
|
72346
|
+
let best = null;
|
|
72347
|
+
for (const t8 of sources) {
|
|
72348
|
+
if (t8.name === self.name || !t8.naturalKey) continue;
|
|
72349
|
+
const p3 = t8.profiles.get(t8.naturalKey);
|
|
72350
|
+
if (!p3 || p3.valueSet.size === 0) continue;
|
|
72351
|
+
let matched = 0;
|
|
72352
|
+
for (const v2 of values) if (p3.valueSet.has(v2)) matched++;
|
|
72353
|
+
if (matched > 0 && (best === null || matched > best.matched)) {
|
|
72354
|
+
best = { target: t8, column: t8.naturalKey, matched };
|
|
72355
|
+
}
|
|
72356
|
+
}
|
|
72357
|
+
return best;
|
|
72358
|
+
}
|
|
72359
|
+
for (const pass of ["array", "scalar"]) {
|
|
72360
|
+
for (const e6 of sources) {
|
|
72361
|
+
for (const [field, p3] of e6.profiles) {
|
|
72362
|
+
if (pass === "array" ? !p3.isArray : p3.isArray) continue;
|
|
72363
|
+
if (pass === "scalar") {
|
|
72364
|
+
if (field === e6.naturalKey) continue;
|
|
72365
|
+
if (FREETEXT.has(normalizeName(field)) || NEVER_KEY.has(normalizeName(field))) continue;
|
|
72366
|
+
if (p3.type !== "text") continue;
|
|
72367
|
+
}
|
|
72368
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
72369
|
+
const best = bestTarget(e6, p3.valueSet);
|
|
72370
|
+
if (!best) continue;
|
|
72371
|
+
const confidence = best.matched / p3.valueSet.size;
|
|
72372
|
+
if (confidence < LINK_MIN_CONFIDENCE) continue;
|
|
72373
|
+
if (linkedTargets.get(e6.name)?.has(best.target.name)) {
|
|
72374
|
+
consume(e6.name, field);
|
|
72375
|
+
continue;
|
|
72376
|
+
}
|
|
72377
|
+
const link = {
|
|
72378
|
+
kind: pass === "array" ? "many-to-many" : "many-to-one",
|
|
72379
|
+
fromEntity: e6.name,
|
|
72380
|
+
fromField: field,
|
|
72381
|
+
toEntity: best.target.name,
|
|
72382
|
+
toKey: normalizeName(best.column),
|
|
72383
|
+
matched: best.matched,
|
|
72384
|
+
unresolved: p3.valueSet.size - best.matched,
|
|
72385
|
+
confidence
|
|
72386
|
+
};
|
|
72387
|
+
if (pass === "array") link.junction = `${e6.name}_${best.target.name}`;
|
|
72388
|
+
linkages.push(link);
|
|
72389
|
+
consume(e6.name, field);
|
|
72390
|
+
markTarget(e6.name, best.target.name);
|
|
72391
|
+
}
|
|
72392
|
+
}
|
|
72393
|
+
}
|
|
72394
|
+
const dimColumnNames = /* @__PURE__ */ new Map();
|
|
72395
|
+
for (const e6 of sources) {
|
|
72396
|
+
for (const [field, p3] of e6.profiles) {
|
|
72397
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
72398
|
+
const nn = normalizeName(field);
|
|
72399
|
+
let arr = dimColumnNames.get(nn);
|
|
72400
|
+
if (!arr) {
|
|
72401
|
+
arr = [];
|
|
72402
|
+
dimColumnNames.set(nn, arr);
|
|
72403
|
+
}
|
|
72404
|
+
arr.push(e6);
|
|
72405
|
+
}
|
|
72406
|
+
}
|
|
72407
|
+
const dimensions = [];
|
|
72408
|
+
const dimByName = /* @__PURE__ */ new Map();
|
|
72409
|
+
for (const e6 of sources) {
|
|
72410
|
+
for (const [field, p3] of e6.profiles) {
|
|
72411
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
72412
|
+
if (field === e6.naturalKey) continue;
|
|
72413
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
72414
|
+
const nn = normalizeName(field);
|
|
72415
|
+
if (FREETEXT.has(nn)) continue;
|
|
72416
|
+
const ratio = p3.distinct / Math.max(1, e6.records.length);
|
|
72417
|
+
const sharedAcross = dimColumnNames.get(nn)?.length ?? 1;
|
|
72418
|
+
const isDim = p3.distinct >= 1 && p3.distinct <= DIM_MAX_DISTINCT && (ratio <= DIM_MAX_RATIO || sharedAcross >= 2);
|
|
72419
|
+
if (!isDim) continue;
|
|
72420
|
+
let dim = dimByName.get(nn);
|
|
72421
|
+
if (!dim) {
|
|
72422
|
+
dim = { name: nn, sourceField: field, fromEntities: [], distinctValues: 0 };
|
|
72423
|
+
dimByName.set(nn, dim);
|
|
72424
|
+
dimensions.push(dim);
|
|
72425
|
+
}
|
|
72426
|
+
if (!dim.fromEntities.includes(e6.name)) dim.fromEntities.push(e6.name);
|
|
72427
|
+
linkages.push({
|
|
72428
|
+
kind: "dimension",
|
|
72429
|
+
fromEntity: e6.name,
|
|
72430
|
+
fromField: field,
|
|
72431
|
+
toEntity: nn,
|
|
72432
|
+
toKey: "value",
|
|
72433
|
+
junction: `${e6.name}_${nn}`,
|
|
72434
|
+
matched: p3.distinct,
|
|
72435
|
+
unresolved: 0,
|
|
72436
|
+
confidence: 1
|
|
72437
|
+
});
|
|
72438
|
+
consume(e6.name, field);
|
|
72439
|
+
}
|
|
72440
|
+
}
|
|
72441
|
+
for (const dim of dimensions) {
|
|
72442
|
+
const all = /* @__PURE__ */ new Set();
|
|
72443
|
+
for (const name of dim.fromEntities) {
|
|
72444
|
+
const e6 = sources.find((s2) => s2.name === name);
|
|
72445
|
+
if (!e6) continue;
|
|
72446
|
+
for (const [f6, p3] of e6.profiles) {
|
|
72447
|
+
if (normalizeName(f6) === dim.name) for (const v2 of p3.valueSet) all.add(v2);
|
|
72448
|
+
}
|
|
72449
|
+
}
|
|
72450
|
+
dim.distinctValues = all.size;
|
|
72451
|
+
}
|
|
72452
|
+
const entities = sources.map((e6) => {
|
|
72453
|
+
const columns = [];
|
|
72454
|
+
for (const [field, p3] of e6.profiles) {
|
|
72455
|
+
if (p3.isArray) continue;
|
|
72456
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
72457
|
+
columns.push({ name: normalizeName(field), sourceKey: field, type: p3.type });
|
|
72458
|
+
}
|
|
72459
|
+
return {
|
|
72460
|
+
name: e6.name,
|
|
72461
|
+
sourceKey: e6.sourceKey,
|
|
72462
|
+
columns,
|
|
72463
|
+
naturalKey: e6.naturalKey ? normalizeName(e6.naturalKey) : null,
|
|
72464
|
+
naturalKeySource: e6.naturalKey,
|
|
72465
|
+
rowCount: e6.records.length,
|
|
72466
|
+
columnar: e6.columnar
|
|
72467
|
+
};
|
|
72468
|
+
});
|
|
72469
|
+
return { entities, dimensions, linkages, skipped };
|
|
72470
|
+
}
|
|
72471
|
+
|
|
72472
|
+
// src/import/dedupe-views.ts
|
|
72473
|
+
init_normalize();
|
|
72474
|
+
var SAMPLE2 = 300;
|
|
72475
|
+
var VIEW_MIN_OVERLAP = 0.8;
|
|
72476
|
+
function buildEntityData(plan, data) {
|
|
72477
|
+
return plan.entities.map((e6) => {
|
|
72478
|
+
const records = sourceRecords(data, e6);
|
|
72479
|
+
const colSet = /* @__PURE__ */ new Set();
|
|
72480
|
+
const colSource = /* @__PURE__ */ new Map();
|
|
72481
|
+
for (const r6 of records.slice(0, SAMPLE2)) {
|
|
72482
|
+
for (const k6 of Object.keys(r6)) {
|
|
72483
|
+
const n3 = normalizeName(k6);
|
|
72484
|
+
colSet.add(n3);
|
|
72485
|
+
if (!colSource.has(n3)) colSource.set(n3, k6);
|
|
72486
|
+
}
|
|
72487
|
+
}
|
|
72488
|
+
const normRows = records.map((r6) => {
|
|
72489
|
+
const o3 = {};
|
|
72490
|
+
for (const k6 of Object.keys(r6)) o3[normalizeName(k6)] = r6[k6];
|
|
72491
|
+
return o3;
|
|
72492
|
+
});
|
|
72493
|
+
return { name: e6.name, sourceKey: e6.sourceKey, cols: [...colSet], colSource, normRows };
|
|
72494
|
+
});
|
|
72495
|
+
}
|
|
72496
|
+
function pickIdentity(a6, shared) {
|
|
72497
|
+
let bestCol = null;
|
|
72498
|
+
let bestDistinct = -1;
|
|
72499
|
+
for (const c6 of shared) {
|
|
72500
|
+
const vals = /* @__PURE__ */ new Set();
|
|
72501
|
+
let textish = 0;
|
|
72502
|
+
let total = 0;
|
|
72503
|
+
for (const r6 of a6.normRows) {
|
|
72504
|
+
const v2 = r6[c6];
|
|
72505
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
72506
|
+
total++;
|
|
72507
|
+
if (typeof v2 === "string") textish++;
|
|
72508
|
+
vals.add(normalizeText(v2));
|
|
72509
|
+
}
|
|
72510
|
+
if (total === 0 || textish / total < 0.7) continue;
|
|
72511
|
+
if (vals.size > bestDistinct) {
|
|
72512
|
+
bestDistinct = vals.size;
|
|
72513
|
+
bestCol = c6;
|
|
72514
|
+
}
|
|
72515
|
+
}
|
|
72516
|
+
return bestCol;
|
|
72517
|
+
}
|
|
72518
|
+
function dedupeAndDetectViews(plan, data) {
|
|
72519
|
+
const entities = buildEntityData(plan, data);
|
|
72520
|
+
const views = [];
|
|
72521
|
+
const asView = /* @__PURE__ */ new Set();
|
|
72522
|
+
const colKeeps = [];
|
|
72523
|
+
for (const a6 of entities) {
|
|
72524
|
+
if (a6.cols.length < 2 || a6.normRows.length === 0) continue;
|
|
72525
|
+
const tabName = normalizeText(a6.sourceKey);
|
|
72526
|
+
if (!tabName) continue;
|
|
72527
|
+
const aColSet = new Set(a6.cols);
|
|
72528
|
+
let best = null;
|
|
72529
|
+
for (const b6 of entities) {
|
|
72530
|
+
if (b6.name === a6.name || asView.has(b6.name)) continue;
|
|
72531
|
+
if (b6.normRows.length < a6.normRows.length) continue;
|
|
72532
|
+
const bColSet = new Set(b6.cols);
|
|
72533
|
+
const shared = a6.cols.filter((c6) => bColSet.has(c6));
|
|
72534
|
+
if (shared.length < Math.max(2, Math.ceil(a6.cols.length * 0.5))) continue;
|
|
72535
|
+
const identity = pickIdentity(a6, shared);
|
|
72536
|
+
if (!identity) continue;
|
|
72537
|
+
const aIds = new Set(
|
|
72538
|
+
a6.normRows.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== "")
|
|
72539
|
+
);
|
|
72540
|
+
if (aIds.size === 0) continue;
|
|
72541
|
+
for (const disc of b6.cols) {
|
|
72542
|
+
if (aColSet.has(disc)) continue;
|
|
72543
|
+
const sub = b6.normRows.filter((r6) => normalizeText(r6[disc]) === tabName);
|
|
72544
|
+
if (sub.length === 0) continue;
|
|
72545
|
+
const bIds = new Set(sub.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== ""));
|
|
72546
|
+
let inter = 0;
|
|
72547
|
+
for (const id of aIds) if (bIds.has(id)) inter++;
|
|
72548
|
+
const overlap = inter / aIds.size;
|
|
72549
|
+
if (overlap < VIEW_MIN_OVERLAP) continue;
|
|
72550
|
+
const rawRow = sub.find((r6) => typeof r6[disc] === "string" || typeof r6[disc] === "number");
|
|
72551
|
+
const raw = rawRow ? rawRow[disc] : void 0;
|
|
72552
|
+
if (typeof raw !== "string" && typeof raw !== "number") continue;
|
|
72553
|
+
if (best === null || overlap > best.overlap || overlap === best.overlap && b6.cols.length > best.master.cols.length) {
|
|
72554
|
+
best = { master: b6, disc, value: String(raw), matched: sub.length, overlap };
|
|
72555
|
+
}
|
|
72556
|
+
}
|
|
72557
|
+
}
|
|
72558
|
+
if (!best) continue;
|
|
72559
|
+
views.push({
|
|
72560
|
+
name: a6.name,
|
|
72561
|
+
master: best.master.name,
|
|
72562
|
+
filterColumn: best.disc,
|
|
72563
|
+
filterValue: best.value,
|
|
72564
|
+
matchedRows: best.matched
|
|
72565
|
+
});
|
|
72566
|
+
asView.add(a6.name);
|
|
72567
|
+
colKeeps.push({ master: best.master, col: best.disc });
|
|
72568
|
+
}
|
|
72569
|
+
for (const { master, col } of colKeeps) {
|
|
72570
|
+
const masterEntity = plan.entities.find((e6) => e6.name === master.name);
|
|
72571
|
+
if (!masterEntity || masterEntity.columns.some((c6) => c6.name === col)) continue;
|
|
72572
|
+
masterEntity.columns.push({
|
|
72573
|
+
name: col,
|
|
72574
|
+
sourceKey: master.colSource.get(col) ?? col,
|
|
72575
|
+
type: inferFieldType(master.normRows.map((r6) => r6[col]))
|
|
72576
|
+
});
|
|
72577
|
+
}
|
|
72578
|
+
if (views.length === 0) return { plan, views };
|
|
72579
|
+
const nextPlan = {
|
|
72580
|
+
entities: plan.entities.filter((e6) => !asView.has(e6.name)),
|
|
72581
|
+
linkages: plan.linkages.filter((l4) => !asView.has(l4.fromEntity)),
|
|
72582
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.filter((n3) => !asView.has(n3)) })).filter((d6) => d6.fromEntities.length > 0),
|
|
72583
|
+
skipped: plan.skipped
|
|
72584
|
+
};
|
|
72585
|
+
return { plan: nextPlan, views };
|
|
72586
|
+
}
|
|
72587
|
+
|
|
72588
|
+
// src/import/excel.ts
|
|
72589
|
+
import { resolve as resolve10 } from "path";
|
|
72590
|
+
var HEADER_SCAN_ROWS = 25;
|
|
72591
|
+
function cellValue(v2) {
|
|
72592
|
+
if (v2 === null || v2 === void 0) return null;
|
|
72593
|
+
if (v2 instanceof Date) return v2.toISOString().slice(0, 10);
|
|
72594
|
+
if (typeof v2 === "object") {
|
|
72595
|
+
const o3 = v2;
|
|
72596
|
+
if ("result" in o3) return cellValue(o3.result);
|
|
72597
|
+
if ("text" in o3) return o3.text;
|
|
72598
|
+
if ("richText" in o3 && Array.isArray(o3.richText)) {
|
|
72599
|
+
return o3.richText.map((t8) => t8.text ?? "").join("");
|
|
72600
|
+
}
|
|
72601
|
+
return null;
|
|
72602
|
+
}
|
|
72603
|
+
return v2;
|
|
72604
|
+
}
|
|
72605
|
+
function isFilled(v2) {
|
|
72606
|
+
return v2 !== null && v2 !== void 0 && v2 !== "";
|
|
72607
|
+
}
|
|
72608
|
+
function sheetToRecords(ws) {
|
|
72609
|
+
const rowCount = ws.rowCount;
|
|
72610
|
+
const colCount = ws.columnCount;
|
|
72611
|
+
if (rowCount < 2 || colCount < 2) return [];
|
|
72612
|
+
const nonEmpty = (r6) => {
|
|
72613
|
+
let n3 = 0;
|
|
72614
|
+
for (let c6 = 1; c6 <= colCount; c6++) if (isFilled(cellValue(ws.getCell(r6, c6).value))) n3++;
|
|
72615
|
+
return n3;
|
|
72616
|
+
};
|
|
72617
|
+
const threshold = Math.max(3, Math.floor(colCount * 0.4));
|
|
72618
|
+
let headerRow = -1;
|
|
72619
|
+
for (let r6 = 1; r6 <= Math.min(HEADER_SCAN_ROWS, rowCount); r6++) {
|
|
72620
|
+
if (nonEmpty(r6) >= threshold && r6 < rowCount && nonEmpty(r6 + 1) >= 2) {
|
|
72621
|
+
headerRow = r6;
|
|
72622
|
+
break;
|
|
72623
|
+
}
|
|
72624
|
+
}
|
|
72625
|
+
if (headerRow < 0) return [];
|
|
72626
|
+
const cols = [];
|
|
72627
|
+
const seen = /* @__PURE__ */ new Set();
|
|
72628
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
72629
|
+
const hv = cellValue(ws.getCell(headerRow, c6).value);
|
|
72630
|
+
if (!isFilled(hv)) continue;
|
|
72631
|
+
const base = String(hv).replace(/\s+/g, " ").trim();
|
|
72632
|
+
if (!base) continue;
|
|
72633
|
+
let name = base;
|
|
72634
|
+
let i6 = 2;
|
|
72635
|
+
while (seen.has(name)) name = base + " " + String(i6++);
|
|
72636
|
+
seen.add(name);
|
|
72637
|
+
cols.push({ c: c6, name });
|
|
72638
|
+
}
|
|
72639
|
+
if (cols.length === 0) return [];
|
|
72640
|
+
const records = [];
|
|
72641
|
+
for (let r6 = headerRow + 1; r6 <= rowCount; r6++) {
|
|
72642
|
+
const row = {};
|
|
72643
|
+
let any = false;
|
|
72644
|
+
for (const { c: c6, name } of cols) {
|
|
72645
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
72646
|
+
if (isFilled(v2)) {
|
|
72647
|
+
row[name] = v2;
|
|
72648
|
+
any = true;
|
|
72649
|
+
}
|
|
72650
|
+
}
|
|
72651
|
+
if (!any) break;
|
|
72652
|
+
const first = cols[0] ? row[cols[0].name] : void 0;
|
|
72653
|
+
if (typeof first === "string" && /^total\b/i.test(first.trim())) continue;
|
|
72654
|
+
records.push(row);
|
|
72655
|
+
}
|
|
72656
|
+
return records;
|
|
72657
|
+
}
|
|
72658
|
+
var preambleCache = /* @__PURE__ */ new Map();
|
|
72659
|
+
function excelPreambleText(absPath) {
|
|
72660
|
+
return preambleCache.get(resolve10(absPath)) ?? "";
|
|
72661
|
+
}
|
|
72662
|
+
function sheetPreamble(ws) {
|
|
72663
|
+
const lines = [];
|
|
72664
|
+
const rowCount = Math.min(10, ws.rowCount);
|
|
72665
|
+
const colCount = Math.min(8, ws.columnCount);
|
|
72666
|
+
for (let r6 = 1; r6 <= rowCount; r6++) {
|
|
72667
|
+
const cells = [];
|
|
72668
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
72669
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
72670
|
+
if (isFilled(v2)) cells.push(String(v2));
|
|
72671
|
+
}
|
|
72672
|
+
if (cells.length) lines.push(cells.join(" "));
|
|
72673
|
+
}
|
|
72674
|
+
return lines.join("\n");
|
|
72675
|
+
}
|
|
72676
|
+
async function excelToRecords(absPath) {
|
|
72677
|
+
let mod;
|
|
72678
|
+
try {
|
|
72679
|
+
mod = await import("exceljs");
|
|
72680
|
+
} catch {
|
|
72681
|
+
throw new Error(
|
|
72682
|
+
'Reading Excel files needs the "exceljs" package \u2014 install it with: npm install exceljs'
|
|
72683
|
+
);
|
|
72684
|
+
}
|
|
72685
|
+
const ExcelJS = mod.default ?? mod;
|
|
72686
|
+
const wb = new ExcelJS.Workbook();
|
|
72687
|
+
await wb.xlsx.readFile(absPath);
|
|
72688
|
+
const out = {};
|
|
72689
|
+
const preamble = [];
|
|
72690
|
+
const props = wb.properties;
|
|
72691
|
+
if (props?.title) preamble.push(props.title);
|
|
72692
|
+
for (const ws of wb.worksheets) {
|
|
72693
|
+
preamble.push(ws.name, sheetPreamble(ws));
|
|
72694
|
+
const records = sheetToRecords(ws);
|
|
72695
|
+
if (records.length > 0) out[ws.name] = records;
|
|
72696
|
+
}
|
|
72697
|
+
preambleCache.set(resolve10(absPath), preamble.filter(Boolean).join("\n"));
|
|
72698
|
+
return out;
|
|
72699
|
+
}
|
|
72700
|
+
|
|
72701
|
+
// src/import/match.ts
|
|
72702
|
+
var BOOKKEEPING = /* @__PURE__ */ new Set(["id", "as_of", "content_key", "deleted_at"]);
|
|
72703
|
+
var MATCH_THRESHOLD = 0.6;
|
|
72704
|
+
function signature(columns) {
|
|
72705
|
+
const out = /* @__PURE__ */ new Set();
|
|
72706
|
+
for (const c6 of columns) {
|
|
72707
|
+
const n3 = normalizeName(c6);
|
|
72708
|
+
if (!n3 || BOOKKEEPING.has(n3) || n3.endsWith("_id")) continue;
|
|
72709
|
+
out.add(n3);
|
|
72710
|
+
}
|
|
72711
|
+
return out;
|
|
72712
|
+
}
|
|
72713
|
+
function containment(a6, b6) {
|
|
72714
|
+
if (a6.size === 0) return 0;
|
|
72715
|
+
let hit = 0;
|
|
72716
|
+
for (const c6 of a6) if (b6.has(c6)) hit++;
|
|
72717
|
+
return hit / a6.size;
|
|
72718
|
+
}
|
|
72719
|
+
function matchSchemaToExisting(existing, plan) {
|
|
72720
|
+
const ex = existing.map((t8) => ({ name: t8.name, sig: signature(t8.columns) }));
|
|
72721
|
+
const matches = [];
|
|
72722
|
+
const rename = {};
|
|
72723
|
+
for (const ent of plan.entities) {
|
|
72724
|
+
const sig = signature(ent.columns.map((c6) => c6.name));
|
|
72725
|
+
if (sig.size === 0) continue;
|
|
72726
|
+
let best = null;
|
|
72727
|
+
for (const t8 of ex) {
|
|
72728
|
+
if (normalizeName(t8.name) === normalizeName(ent.name)) {
|
|
72729
|
+
best = { name: t8.name, overlap: 1 };
|
|
72730
|
+
break;
|
|
72731
|
+
}
|
|
72732
|
+
const overlap = containment(sig, t8.sig);
|
|
72733
|
+
if (overlap > (best?.overlap ?? 0)) best = { name: t8.name, overlap };
|
|
72734
|
+
}
|
|
72735
|
+
if (best && best.overlap >= MATCH_THRESHOLD) {
|
|
72736
|
+
matches.push({ from: ent.name, to: best.name, overlap: best.overlap });
|
|
72737
|
+
if (best.name !== ent.name) rename[ent.name] = best.name;
|
|
72738
|
+
}
|
|
72739
|
+
}
|
|
72740
|
+
const totalEntities = plan.entities.length;
|
|
72741
|
+
const matchedCount = matches.length;
|
|
72742
|
+
const isKnownDocument = totalEntities > 0 && matchedCount >= Math.ceil(totalEntities / 2);
|
|
72743
|
+
return { matches, rename, matchedCount, totalEntities, isKnownDocument };
|
|
72744
|
+
}
|
|
72745
|
+
function renameEntities(plan, views, rename) {
|
|
72746
|
+
if (Object.keys(rename).length === 0) return { plan, views };
|
|
72747
|
+
const r6 = (n3) => rename[n3] ?? n3;
|
|
72748
|
+
return {
|
|
72749
|
+
plan: {
|
|
72750
|
+
...plan,
|
|
72751
|
+
entities: plan.entities.map((e6) => ({ ...e6, name: r6(e6.name) })),
|
|
72752
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.map(r6) })),
|
|
72753
|
+
linkages: plan.linkages.map((l4) => ({
|
|
72754
|
+
...l4,
|
|
72755
|
+
fromEntity: r6(l4.fromEntity),
|
|
72756
|
+
toEntity: r6(l4.toEntity),
|
|
72757
|
+
...l4.junction ? { junction: l4.junction } : {}
|
|
72758
|
+
}))
|
|
72759
|
+
},
|
|
72760
|
+
views: views.map((v2) => ({ ...v2, name: r6(v2.name), master: r6(v2.master) }))
|
|
72761
|
+
};
|
|
72762
|
+
}
|
|
72763
|
+
|
|
72764
|
+
// src/import/materialize.ts
|
|
72765
|
+
init_parser();
|
|
72766
|
+
import { createHash as createHash11 } from "crypto";
|
|
72767
|
+
import { existsSync as existsSync26 } from "fs";
|
|
72768
|
+
init_normalize();
|
|
72769
|
+
|
|
72770
|
+
// src/import/asof.ts
|
|
72771
|
+
var MONTHS2 = {
|
|
72772
|
+
jan: 1,
|
|
72773
|
+
january: 1,
|
|
72774
|
+
feb: 2,
|
|
72775
|
+
february: 2,
|
|
72776
|
+
mar: 3,
|
|
72777
|
+
march: 3,
|
|
72778
|
+
apr: 4,
|
|
72779
|
+
april: 4,
|
|
72780
|
+
may: 5,
|
|
72781
|
+
jun: 6,
|
|
72782
|
+
june: 6,
|
|
72783
|
+
jul: 7,
|
|
72784
|
+
july: 7,
|
|
72785
|
+
aug: 8,
|
|
72786
|
+
august: 8,
|
|
72787
|
+
sep: 9,
|
|
72788
|
+
sept: 9,
|
|
72789
|
+
september: 9,
|
|
72790
|
+
oct: 10,
|
|
72791
|
+
october: 10,
|
|
72792
|
+
nov: 11,
|
|
72793
|
+
november: 11,
|
|
72794
|
+
dec: 12,
|
|
72795
|
+
december: 12
|
|
72796
|
+
};
|
|
72797
|
+
var ASOF_KEYWORDS = /\b(as[ -]?of|as at|period (?:end(?:ed|ing)?|of)|fye|fiscal year end(?:ed|ing)?|year[ -]?end(?:ed|ing)?|quarter[ -]?end(?:ed|ing)?|valuation date|report(?:ing)? date|effective date|dated)\b/i;
|
|
72798
|
+
function isoFrom(y2, m4, d6) {
|
|
72799
|
+
if (m4 < 1 || m4 > 12 || d6 < 1 || d6 > 31) return null;
|
|
72800
|
+
if (y2 < 2010 || y2 > 2099) return null;
|
|
72801
|
+
return `${String(y2)}-${String(m4).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
72802
|
+
}
|
|
72803
|
+
function findDates(text) {
|
|
72804
|
+
const hits = [];
|
|
72805
|
+
const push = (date2, match, index) => {
|
|
72806
|
+
if (date2) hits.push({ date: date2, match, index });
|
|
72807
|
+
};
|
|
72808
|
+
for (const m4 of text.matchAll(/(20\d{2})[-._/](\d{1,2})[-._/](\d{1,2})/g)) {
|
|
72809
|
+
push(isoFrom(Number(m4[1]), Number(m4[2]), Number(m4[3])), m4[0], m4.index);
|
|
72810
|
+
}
|
|
72811
|
+
for (const m4 of text.matchAll(/(\d{1,2})[-._/](\d{1,2})[-._/](\d{2,4})/g)) {
|
|
72812
|
+
let y2 = Number(m4[3]);
|
|
72813
|
+
if (y2 < 100) y2 += 2e3;
|
|
72814
|
+
push(isoFrom(y2, Number(m4[1]), Number(m4[2])), m4[0], m4.index);
|
|
72815
|
+
}
|
|
72816
|
+
for (const m4 of text.matchAll(/([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(20\d{2})/g)) {
|
|
72817
|
+
const mon = MONTHS2[(m4[1] ?? "").toLowerCase()];
|
|
72818
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[2])), m4[0], m4.index);
|
|
72819
|
+
}
|
|
72820
|
+
for (const m4 of text.matchAll(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(20\d{2})/g)) {
|
|
72821
|
+
const mon = MONTHS2[(m4[2] ?? "").toLowerCase()];
|
|
72822
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[1])), m4[0], m4.index);
|
|
72823
|
+
}
|
|
72824
|
+
return hits;
|
|
72825
|
+
}
|
|
72826
|
+
function parseCellDate(value) {
|
|
72827
|
+
if (value instanceof Date) {
|
|
72828
|
+
return isoFrom(value.getUTCFullYear(), value.getUTCMonth() + 1, value.getUTCDate());
|
|
72829
|
+
}
|
|
72830
|
+
if (typeof value === "string") return findDates(value)[0]?.date ?? null;
|
|
72831
|
+
return null;
|
|
72832
|
+
}
|
|
72833
|
+
function scanText(text, label) {
|
|
72834
|
+
if (!text) return [];
|
|
72835
|
+
const out = [];
|
|
72836
|
+
for (const hit of findDates(text)) {
|
|
72837
|
+
const before = text.slice(Math.max(0, hit.index - 40), hit.index);
|
|
72838
|
+
const keyworded = ASOF_KEYWORDS.test(before) || ASOF_KEYWORDS.test(hit.match);
|
|
72839
|
+
const snippet = text.slice(Math.max(0, hit.index - 24), hit.index + hit.match.length + 4).replace(/\s+/g, " ").trim();
|
|
72840
|
+
out.push({
|
|
72841
|
+
date: hit.date,
|
|
72842
|
+
source: "content",
|
|
72843
|
+
confidence: keyworded ? 0.95 : 0.7,
|
|
72844
|
+
evidence: `${label}: "${snippet}"`
|
|
72845
|
+
});
|
|
72846
|
+
}
|
|
72847
|
+
return out;
|
|
72848
|
+
}
|
|
72849
|
+
function scanFilename(fileName) {
|
|
72850
|
+
if (!fileName) return [];
|
|
72851
|
+
const base = fileName.replace(/\.[A-Za-z0-9]+$/, "");
|
|
72852
|
+
return findDates(base).map((hit, i6, all) => ({
|
|
72853
|
+
date: hit.date,
|
|
72854
|
+
source: "filename",
|
|
72855
|
+
confidence: i6 === all.length - 1 ? 0.6 : 0.45,
|
|
72856
|
+
evidence: `file name: "${hit.match}"`
|
|
72857
|
+
}));
|
|
72858
|
+
}
|
|
72859
|
+
function detectAsOfCandidates(inputs) {
|
|
72860
|
+
const all = [];
|
|
72861
|
+
for (const t8 of inputs.texts ?? []) all.push(...scanText(t8.text, t8.label));
|
|
72862
|
+
if (inputs.fileName) all.push(...scanFilename(inputs.fileName));
|
|
72863
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
72864
|
+
for (const c6 of all) {
|
|
72865
|
+
const prev = byDate.get(c6.date);
|
|
72866
|
+
if (!prev || c6.confidence > prev.confidence) byDate.set(c6.date, c6);
|
|
72867
|
+
}
|
|
72868
|
+
return [...byDate.values()].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
72869
|
+
}
|
|
72870
|
+
|
|
72871
|
+
// src/import/materialize.ts
|
|
72872
|
+
function coerce2(v2, type) {
|
|
72873
|
+
if (v2 === null || v2 === void 0 || v2 === "") return null;
|
|
72874
|
+
if (type === "boolean") return v2 === true || v2 === "true" || v2 === 1 ? 1 : 0;
|
|
72875
|
+
return v2;
|
|
72876
|
+
}
|
|
72877
|
+
function contentKey(record) {
|
|
72878
|
+
const parts = Object.keys(record).sort().map((k6) => k6 + "=" + JSON.stringify(record[k6] ?? null));
|
|
72879
|
+
return createHash11("sha256").update(parts.join("|")).digest("hex");
|
|
72880
|
+
}
|
|
72881
|
+
function persistTable(configPath, name, fields) {
|
|
72882
|
+
if (!configPath || !existsSync26(configPath)) return;
|
|
72883
|
+
try {
|
|
72884
|
+
const doc = loadConfigDoc(configPath);
|
|
72885
|
+
doc.setIn(["entities", name], { fields, outputFile: name.toUpperCase() + ".md" });
|
|
72886
|
+
saveConfigDoc(configPath, doc);
|
|
72887
|
+
} catch {
|
|
72888
|
+
}
|
|
72889
|
+
}
|
|
72890
|
+
async function materializeImport(ctx, data, plan, views = [], opts = {}) {
|
|
72891
|
+
const { db, configPath } = ctx;
|
|
72892
|
+
const mode = opts.mode ?? "both";
|
|
72893
|
+
const doSchema = mode === "schema" || mode === "both";
|
|
72894
|
+
const doContents = mode === "contents" || mode === "both";
|
|
72895
|
+
const asOf = opts.asOf?.trim() ? opts.asOf.trim() : null;
|
|
72896
|
+
const asOfColumn = opts.asOfColumn?.trim() ? opts.asOfColumn.trim() : null;
|
|
72897
|
+
const dated = asOf !== null || asOfColumn !== null;
|
|
72898
|
+
const asOfSourceKey = (entity) => asOfColumn ? entity.columns.find((c6) => c6.name === asOfColumn)?.sourceKey ?? null : null;
|
|
72899
|
+
const rowAsOf = (entity, record) => {
|
|
72900
|
+
const sk = asOfSourceKey(entity);
|
|
72901
|
+
if (sk) {
|
|
72902
|
+
const d6 = parseCellDate(record[sk]);
|
|
72903
|
+
if (d6) return d6;
|
|
72904
|
+
}
|
|
72905
|
+
return asOf;
|
|
72906
|
+
};
|
|
72907
|
+
const recordKey = (entity, record) => {
|
|
72908
|
+
const a6 = rowAsOf(entity, record);
|
|
72909
|
+
return a6 ? contentKey({ ...record, __as_of: a6 }) : contentKey(record);
|
|
72910
|
+
};
|
|
72911
|
+
const scopedKey = (a6, keyVal) => (a6 ?? "") + "|" + normalizeText(keyVal);
|
|
72912
|
+
const report = async (p3) => {
|
|
72913
|
+
await opts.onProgress?.(p3);
|
|
72914
|
+
};
|
|
72915
|
+
const tablesCreated = [];
|
|
72916
|
+
const rowsByTable = {};
|
|
72917
|
+
const links = [];
|
|
72918
|
+
const viewResults = [];
|
|
72919
|
+
const byName = new Map(plan.entities.map((e6) => [e6.name, e6]));
|
|
72920
|
+
for (const entity of plan.entities) {
|
|
72921
|
+
const keyless = entity.naturalKey === null;
|
|
72922
|
+
const columns = { id: "TEXT PRIMARY KEY" };
|
|
72923
|
+
const fieldTypes = {};
|
|
72924
|
+
const cfgFields = { id: { type: "uuid", primaryKey: true } };
|
|
72925
|
+
for (const c6 of entity.columns) {
|
|
72926
|
+
columns[c6.name] = fieldToSqliteBaseType(c6.type);
|
|
72927
|
+
fieldTypes[c6.name] = c6.type;
|
|
72928
|
+
cfgFields[c6.name] = { type: c6.type };
|
|
72929
|
+
}
|
|
72930
|
+
const needsContentKey = keyless || dated;
|
|
72931
|
+
if (needsContentKey) {
|
|
72932
|
+
columns.content_key = "TEXT";
|
|
72933
|
+
cfgFields.content_key = { type: "text" };
|
|
72934
|
+
}
|
|
72935
|
+
if (dated) {
|
|
72936
|
+
columns.as_of = "TEXT";
|
|
72937
|
+
cfgFields.as_of = { type: "text" };
|
|
72938
|
+
}
|
|
72939
|
+
columns.deleted_at = "TEXT";
|
|
72940
|
+
cfgFields.deleted_at = { type: "text" };
|
|
72941
|
+
if (!db.getRegisteredTableNames().includes(entity.name)) tablesCreated.push(entity.name);
|
|
72942
|
+
await db.defineLate(entity.name, { columns, fieldTypes, primaryKey: "id" });
|
|
72943
|
+
persistTable(configPath, entity.name, cfgFields);
|
|
72944
|
+
await report({
|
|
72945
|
+
phase: "entities",
|
|
72946
|
+
table: entity.name,
|
|
72947
|
+
message: `Created table ${entity.name}`
|
|
72948
|
+
});
|
|
72949
|
+
if (doContents) {
|
|
72950
|
+
const records = sourceRecords(data, entity);
|
|
72951
|
+
const rows = records.map((r6) => {
|
|
72952
|
+
const row = {};
|
|
72953
|
+
for (const c6 of entity.columns) row[c6.name] = coerce2(r6[c6.sourceKey], c6.type);
|
|
72954
|
+
if (needsContentKey) row.content_key = recordKey(entity, r6);
|
|
72955
|
+
if (dated) row.as_of = rowAsOf(entity, r6);
|
|
72956
|
+
return row;
|
|
72957
|
+
});
|
|
72958
|
+
await db.seed({
|
|
72959
|
+
data: rows,
|
|
72960
|
+
table: entity.name,
|
|
72961
|
+
naturalKey: dated ? "content_key" : entity.naturalKey ?? "content_key"
|
|
72962
|
+
});
|
|
72963
|
+
const n3 = await db.count(entity.name);
|
|
72964
|
+
rowsByTable[entity.name] = n3;
|
|
72965
|
+
await report({
|
|
72966
|
+
phase: "entities",
|
|
72967
|
+
table: entity.name,
|
|
72968
|
+
count: n3,
|
|
72969
|
+
message: `Loaded ${String(n3)} rows into ${entity.name}`
|
|
72970
|
+
});
|
|
72971
|
+
}
|
|
72972
|
+
}
|
|
72973
|
+
for (const dim of plan.dimensions) {
|
|
72974
|
+
if (!db.getRegisteredTableNames().includes(dim.name)) tablesCreated.push(dim.name);
|
|
72975
|
+
await db.defineLate(dim.name, {
|
|
72976
|
+
columns: { id: "TEXT PRIMARY KEY", value: "TEXT", deleted_at: "TEXT" },
|
|
72977
|
+
fieldTypes: { value: "text" },
|
|
72978
|
+
primaryKey: "id"
|
|
72979
|
+
});
|
|
72980
|
+
persistTable(configPath, dim.name, {
|
|
72981
|
+
id: { type: "uuid", primaryKey: true },
|
|
72982
|
+
value: { type: "text" },
|
|
72983
|
+
deleted_at: { type: "text" }
|
|
72984
|
+
});
|
|
72985
|
+
if (doSchema) {
|
|
72986
|
+
const values = /* @__PURE__ */ new Map();
|
|
72987
|
+
for (const ename of dim.fromEntities) {
|
|
72988
|
+
const ent = byName.get(ename);
|
|
72989
|
+
if (!ent) continue;
|
|
72990
|
+
const records = sourceRecords(data, ent);
|
|
72991
|
+
const first = records[0];
|
|
72992
|
+
const srcKey = first ? Object.keys(first).find((k6) => normalizeName(k6) === dim.name) : void 0;
|
|
72993
|
+
if (!srcKey) continue;
|
|
72994
|
+
for (const r6 of records) {
|
|
72995
|
+
const v2 = r6[srcKey];
|
|
72996
|
+
if (typeof v2 !== "string" && typeof v2 !== "number") continue;
|
|
72997
|
+
const key = normalizeText(v2);
|
|
72998
|
+
if (key !== "" && !values.has(key)) values.set(key, String(v2));
|
|
72999
|
+
}
|
|
73000
|
+
}
|
|
73001
|
+
await db.seed({
|
|
73002
|
+
data: [...values.values()].map((value) => ({ value })),
|
|
73003
|
+
table: dim.name,
|
|
73004
|
+
naturalKey: "value"
|
|
73005
|
+
});
|
|
73006
|
+
const n3 = await db.count(dim.name);
|
|
73007
|
+
rowsByTable[dim.name] = n3;
|
|
73008
|
+
await report({
|
|
73009
|
+
phase: "dimensions",
|
|
73010
|
+
table: dim.name,
|
|
73011
|
+
count: n3,
|
|
73012
|
+
message: `Dimension ${dim.name}: ${String(n3)} values`
|
|
73013
|
+
});
|
|
73014
|
+
}
|
|
73015
|
+
}
|
|
73016
|
+
const idMapCache = /* @__PURE__ */ new Map();
|
|
73017
|
+
async function idMap(table, keyCol, datedTarget) {
|
|
73018
|
+
const cacheKey = table + ":" + keyCol + ":" + (datedTarget ? "D" : "");
|
|
73019
|
+
const cached = idMapCache.get(cacheKey);
|
|
73020
|
+
if (cached) return cached;
|
|
73021
|
+
const map = /* @__PURE__ */ new Map();
|
|
73022
|
+
for (const r6 of await db.query(table)) {
|
|
73023
|
+
const k6 = r6[keyCol];
|
|
73024
|
+
if (k6 === null || k6 === void 0) continue;
|
|
73025
|
+
const mapKey = datedTarget ? scopedKey(r6.as_of, k6) : normalizeText(k6);
|
|
73026
|
+
map.set(mapKey, String(r6.id));
|
|
73027
|
+
}
|
|
73028
|
+
idMapCache.set(cacheKey, map);
|
|
73029
|
+
return map;
|
|
73030
|
+
}
|
|
73031
|
+
for (const link of plan.linkages) {
|
|
73032
|
+
const from = byName.get(link.fromEntity);
|
|
73033
|
+
if (!from) continue;
|
|
73034
|
+
const jName = link.junction ?? `${link.fromEntity}_${link.toEntity}`;
|
|
73035
|
+
const fromFk = `${link.fromEntity}_id`;
|
|
73036
|
+
const toFk = `${link.toEntity}_id`;
|
|
73037
|
+
const jCols = {
|
|
73038
|
+
id: "TEXT PRIMARY KEY",
|
|
73039
|
+
[fromFk]: "TEXT",
|
|
73040
|
+
[toFk]: "TEXT"
|
|
73041
|
+
};
|
|
73042
|
+
const jCfg = {
|
|
73043
|
+
id: { type: "uuid", primaryKey: true },
|
|
73044
|
+
[fromFk]: { type: "uuid", ref: link.fromEntity },
|
|
73045
|
+
[toFk]: { type: "uuid", ref: link.toEntity }
|
|
73046
|
+
};
|
|
73047
|
+
if (dated) {
|
|
73048
|
+
jCols.as_of = "TEXT";
|
|
73049
|
+
jCfg.as_of = { type: "text" };
|
|
73050
|
+
}
|
|
73051
|
+
if (!db.getRegisteredTableNames().includes(jName)) tablesCreated.push(jName);
|
|
73052
|
+
await db.defineLate(jName, { columns: jCols, primaryKey: "id" });
|
|
73053
|
+
persistTable(configPath, jName, jCfg);
|
|
73054
|
+
if (!doContents) continue;
|
|
73055
|
+
const fromKeyCol = from.naturalKey ?? "content_key";
|
|
73056
|
+
const toIsEntity = byName.has(link.toEntity);
|
|
73057
|
+
const fromMap = await idMap(link.fromEntity, fromKeyCol, dated);
|
|
73058
|
+
const toMap = await idMap(link.toEntity, link.toKey, toIsEntity && dated);
|
|
73059
|
+
const seen = /* @__PURE__ */ new Set();
|
|
73060
|
+
for (const r6 of await db.query(jName)) {
|
|
73061
|
+
seen.add(String(r6[fromFk]) + "|" + String(r6[toFk]));
|
|
73062
|
+
}
|
|
73063
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
73064
|
+
let created = 0;
|
|
73065
|
+
for (const record of sourceRecords(data, from)) {
|
|
73066
|
+
const a6 = rowAsOf(from, record);
|
|
73067
|
+
const fromKeyVal = from.naturalKey === null ? recordKey(from, record) : record[from.naturalKeySource ?? ""];
|
|
73068
|
+
const fromId = fromMap.get(dated ? scopedKey(a6, fromKeyVal) : normalizeText(fromKeyVal));
|
|
73069
|
+
if (!fromId) continue;
|
|
73070
|
+
const raw = record[link.fromField];
|
|
73071
|
+
const refs = Array.isArray(raw) ? raw : [raw];
|
|
73072
|
+
for (const ref of refs) {
|
|
73073
|
+
if (ref === null || ref === void 0 || ref === "") continue;
|
|
73074
|
+
const toId = toMap.get(toIsEntity && dated ? scopedKey(a6, ref) : normalizeText(ref));
|
|
73075
|
+
if (!toId) {
|
|
73076
|
+
unresolved.add(normalizeText(ref));
|
|
73077
|
+
continue;
|
|
73078
|
+
}
|
|
73079
|
+
const edge = fromId + "|" + toId;
|
|
73080
|
+
if (seen.has(edge)) continue;
|
|
73081
|
+
seen.add(edge);
|
|
73082
|
+
await db.insert(
|
|
73083
|
+
jName,
|
|
73084
|
+
dated ? { [fromFk]: fromId, [toFk]: toId, as_of: a6 } : { [fromFk]: fromId, [toFk]: toId }
|
|
73085
|
+
);
|
|
73086
|
+
created++;
|
|
73087
|
+
}
|
|
73088
|
+
}
|
|
73089
|
+
rowsByTable[jName] = created;
|
|
73090
|
+
links.push({ junction: jName, created, unresolved: unresolved.size });
|
|
73091
|
+
await report({
|
|
73092
|
+
phase: "links",
|
|
73093
|
+
table: jName,
|
|
73094
|
+
count: created,
|
|
73095
|
+
message: `Linked ${String(created)} ${jName}`
|
|
73096
|
+
});
|
|
73097
|
+
}
|
|
73098
|
+
if (doSchema) {
|
|
73099
|
+
for (const v2 of views) {
|
|
73100
|
+
const filt = v2.filterValue.replace(/'/g, "''");
|
|
73101
|
+
await execSql(db, `DROP VIEW IF EXISTS "${v2.name}"`);
|
|
73102
|
+
await execSql(
|
|
73103
|
+
db,
|
|
73104
|
+
`CREATE VIEW "${v2.name}" AS SELECT * FROM "${v2.master}" WHERE "${v2.filterColumn}" = '${filt}'`
|
|
73105
|
+
);
|
|
73106
|
+
const cols = await db.introspectColumns(v2.name);
|
|
73107
|
+
await db.defineLate(v2.name, {
|
|
73108
|
+
columns: Object.fromEntries(cols.map((c6) => [c6, "TEXT"])),
|
|
73109
|
+
render: () => ""
|
|
73110
|
+
});
|
|
73111
|
+
if (!tablesCreated.includes(v2.name)) tablesCreated.push(v2.name);
|
|
73112
|
+
const rows = await db.count(v2.name);
|
|
73113
|
+
rowsByTable[v2.name] = rows;
|
|
73114
|
+
viewResults.push({ name: v2.name, master: v2.master, rows });
|
|
73115
|
+
await report({
|
|
73116
|
+
phase: "views",
|
|
73117
|
+
table: v2.name,
|
|
73118
|
+
count: rows,
|
|
73119
|
+
message: `View ${v2.name}: ${String(rows)} rows`
|
|
73120
|
+
});
|
|
73121
|
+
}
|
|
73122
|
+
}
|
|
73123
|
+
await report({ phase: "done", message: "Import complete" });
|
|
73124
|
+
return { mode, asOf, asOfColumn, tablesCreated, rowsByTable, links, views: viewResults };
|
|
73125
|
+
}
|
|
73126
|
+
|
|
73127
|
+
// src/gui/import-auto.ts
|
|
73128
|
+
init_native_entities();
|
|
73129
|
+
|
|
73130
|
+
// src/gui/import-detect.ts
|
|
73131
|
+
import { basename as basename10 } from "path";
|
|
73132
|
+
|
|
73133
|
+
// src/gui/ai/asof-llm.ts
|
|
73134
|
+
init_assistant_routes();
|
|
73135
|
+
init_chat();
|
|
73136
|
+
var MAX_CHARS = 6e3;
|
|
73137
|
+
var SYSTEM = 'You extract the single "as of" / report / snapshot / period-end date from the text of a data file (a financial statement, track record, export, etc.). Reply with ONLY that date as ISO YYYY-MM-DD, or the exact word NONE if the text has no such date. Output nothing else \u2014 no prose, no quotes.';
|
|
73138
|
+
function parseLlmDate(reply) {
|
|
73139
|
+
if (!reply) return null;
|
|
73140
|
+
const m4 = /(20\d{2})-(\d{2})-(\d{2})/.exec(reply);
|
|
73141
|
+
if (!m4) return null;
|
|
73142
|
+
const y2 = Number(m4[1]);
|
|
73143
|
+
const mo = Number(m4[2]);
|
|
73144
|
+
const d6 = Number(m4[3]);
|
|
73145
|
+
if (mo < 1 || mo > 12 || d6 < 1 || d6 > 31 || y2 < 2010 || y2 > 2099) return null;
|
|
73146
|
+
return `${String(y2)}-${String(mo).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
73147
|
+
}
|
|
73148
|
+
async function asOfFromLlm(db, text) {
|
|
73149
|
+
const trimmed = text.trim();
|
|
73150
|
+
if (!trimmed) return null;
|
|
73151
|
+
try {
|
|
73152
|
+
const auth = await resolveClaudeAuth(db);
|
|
73153
|
+
if (!auth) return null;
|
|
73154
|
+
const client = createAnthropicClient(auth);
|
|
73155
|
+
const result = await client.runTurn({
|
|
73156
|
+
model: DEFAULT_MODEL2,
|
|
73157
|
+
system: SYSTEM,
|
|
73158
|
+
temperature: 0,
|
|
73159
|
+
tools: [],
|
|
73160
|
+
messages: [{ role: "user", content: `File text:
|
|
73161
|
+
${trimmed.slice(0, MAX_CHARS)}` }],
|
|
73162
|
+
onText: () => {
|
|
73163
|
+
}
|
|
73164
|
+
});
|
|
73165
|
+
const date2 = parseLlmDate(result.text);
|
|
73166
|
+
return date2 ? { date: date2, source: "llm", confidence: 0.85, evidence: "Claude read the file" } : null;
|
|
73167
|
+
} catch (e6) {
|
|
73168
|
+
console.warn("[import] as-of LLM fallback failed:", e6.message);
|
|
73169
|
+
return null;
|
|
73170
|
+
}
|
|
73171
|
+
}
|
|
73172
|
+
|
|
73173
|
+
// src/gui/import-detect.ts
|
|
73174
|
+
async function detectImportAsOf(db, data, opts = {}) {
|
|
73175
|
+
const fileName = opts.fileName ?? (opts.abs ? basename10(opts.abs).replace(/^[0-9a-f]{8}-/, "") : "");
|
|
73176
|
+
const texts = [];
|
|
73177
|
+
for (const [k6, v2] of Object.entries(data)) {
|
|
73178
|
+
if (!Array.isArray(v2)) texts.push({ label: "data", text: `${k6}: ${JSON.stringify(v2)}` });
|
|
73179
|
+
}
|
|
73180
|
+
if (opts.abs && /\.xlsx?$/i.test(opts.abs)) {
|
|
73181
|
+
const pre = excelPreambleText(opts.abs);
|
|
73182
|
+
if (pre) texts.push({ label: "title", text: pre });
|
|
73183
|
+
}
|
|
73184
|
+
let candidates = detectAsOfCandidates({ fileName, texts });
|
|
73185
|
+
if (!candidates[0] || candidates[0].confidence < 0.7) {
|
|
73186
|
+
const llm = await asOfFromLlm(db, texts.map((t8) => t8.text).join("\n"));
|
|
73187
|
+
if (llm) candidates = [...candidates, llm].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
73188
|
+
}
|
|
73189
|
+
return candidates;
|
|
73190
|
+
}
|
|
73191
|
+
|
|
73192
|
+
// src/import/asof-columns.ts
|
|
73193
|
+
var STRONG_NAME = /(as[_ -]?of|as[_ -]?at|report(?:ing)?[_ -]?date|valuation[_ -]?date|effective[_ -]?date|period[_ -]?end|snapshot[_ -]?date|statement[_ -]?date|fye)/i;
|
|
73194
|
+
var WEAK_NAME = /(^|_)(date|period|quarter|asof)($|_)/i;
|
|
73195
|
+
function detectAsOfColumns(data, plan) {
|
|
73196
|
+
const out = [];
|
|
73197
|
+
for (const entity of plan.entities) {
|
|
73198
|
+
const records = sourceRecords(data, entity);
|
|
73199
|
+
if (records.length < 2) continue;
|
|
73200
|
+
for (const col of entity.columns) {
|
|
73201
|
+
const strong = STRONG_NAME.test(col.name);
|
|
73202
|
+
const weak = WEAK_NAME.test(col.name);
|
|
73203
|
+
if (!strong && !weak) continue;
|
|
73204
|
+
const vals = records.map((r6) => r6[col.sourceKey]).filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
73205
|
+
if (vals.length < Math.max(3, Math.floor(records.length * 0.5))) continue;
|
|
73206
|
+
const dates = vals.map(parseCellDate).filter((d6) => d6 !== null);
|
|
73207
|
+
if (dates.length / vals.length < 0.8) continue;
|
|
73208
|
+
const distinctDates = new Set(dates).size;
|
|
73209
|
+
const typed = col.type === "date" || col.type === "datetime";
|
|
73210
|
+
let confidence = strong ? 0.9 : 0.6;
|
|
73211
|
+
if (typed) confidence += 0.03;
|
|
73212
|
+
if (distinctDates > 1) confidence += 0.04;
|
|
73213
|
+
out.push({
|
|
73214
|
+
entity: entity.name,
|
|
73215
|
+
column: col.name,
|
|
73216
|
+
confidence: Math.min(confidence, 0.97),
|
|
73217
|
+
distinctDates,
|
|
73218
|
+
evidence: `column "${col.name}" \u2014 ${String(distinctDates)} distinct date${distinctDates === 1 ? "" : "s"} across ${String(vals.length)} rows`
|
|
73219
|
+
});
|
|
73220
|
+
}
|
|
73221
|
+
}
|
|
73222
|
+
return out.sort((a6, b6) => b6.confidence - a6.confidence);
|
|
73223
|
+
}
|
|
73224
|
+
|
|
73225
|
+
// src/gui/import-auto.ts
|
|
73226
|
+
function existingDataTables(db) {
|
|
73227
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
73228
|
+
const out = [];
|
|
73229
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
73230
|
+
if (native.has(t8)) continue;
|
|
73231
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
73232
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
73233
|
+
}
|
|
73234
|
+
return out;
|
|
73235
|
+
}
|
|
73236
|
+
async function readStructured(abs, name) {
|
|
73237
|
+
if (/\.xlsx?$/i.test(name)) return excelToRecords(abs);
|
|
73238
|
+
return JSON.parse(readFileSync22(abs, "utf8"));
|
|
73239
|
+
}
|
|
73240
|
+
async function autoImportStructured(db, configPath, abs, name) {
|
|
73241
|
+
if (!/\.(xlsx?|json)$/i.test(name)) return null;
|
|
73242
|
+
let data;
|
|
73243
|
+
try {
|
|
73244
|
+
data = await readStructured(abs, name);
|
|
73245
|
+
} catch {
|
|
73246
|
+
return null;
|
|
73247
|
+
}
|
|
73248
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
73249
|
+
inferSchema(data),
|
|
73250
|
+
data
|
|
73251
|
+
);
|
|
73252
|
+
if (inferredPlan.entities.length === 0) return null;
|
|
73253
|
+
const schemaMatch = matchSchemaToExisting(existingDataTables(db), inferredPlan);
|
|
73254
|
+
const asOfCandidates = await detectImportAsOf(db, data, { abs, fileName: name });
|
|
73255
|
+
const asOf = asOfCandidates[0]?.date ?? null;
|
|
73256
|
+
const asOfColumns = detectAsOfColumns(data, inferredPlan);
|
|
73257
|
+
const proposal = {
|
|
73258
|
+
plan: inferredPlan,
|
|
73259
|
+
views: inferredViews,
|
|
73260
|
+
asOfCandidates,
|
|
73261
|
+
asOfColumns,
|
|
73262
|
+
schemaMatch,
|
|
73263
|
+
matchedCount: schemaMatch.matchedCount,
|
|
73264
|
+
totalEntities: schemaMatch.totalEntities,
|
|
73265
|
+
tables: [],
|
|
73266
|
+
rows: 0
|
|
73267
|
+
};
|
|
73268
|
+
if (!schemaMatch.isKnownDocument) {
|
|
73269
|
+
return { imported: false, reason: "new-dataset", asOf, ...proposal };
|
|
73270
|
+
}
|
|
73271
|
+
if (!asOf) {
|
|
73272
|
+
return { imported: false, reason: "needs-confirm", asOf: null, ...proposal };
|
|
73273
|
+
}
|
|
73274
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, schemaMatch.rename);
|
|
73275
|
+
const result = await materializeImport({ db, configPath }, data, plan, views, { asOf });
|
|
73276
|
+
const rows = Object.values(result.rowsByTable).reduce((a6, b6) => a6 + b6, 0);
|
|
73277
|
+
return {
|
|
73278
|
+
imported: true,
|
|
73279
|
+
asOf,
|
|
73280
|
+
matchedCount: schemaMatch.matchedCount,
|
|
73281
|
+
totalEntities: schemaMatch.totalEntities,
|
|
73282
|
+
tables: Object.keys(result.rowsByTable),
|
|
73283
|
+
rows
|
|
73284
|
+
};
|
|
73285
|
+
}
|
|
73286
|
+
|
|
73287
|
+
// src/gui/ingest-routes.ts
|
|
71598
73288
|
var MIME_BY_EXT = {
|
|
71599
73289
|
".pdf": "application/pdf",
|
|
71600
73290
|
".png": "image/png",
|
|
@@ -71696,28 +73386,28 @@ ${err.stack ?? ""}`
|
|
|
71696
73386
|
return null;
|
|
71697
73387
|
}
|
|
71698
73388
|
}
|
|
71699
|
-
async function extractImage(db,
|
|
73389
|
+
async function extractImage(db, path3, mime) {
|
|
71700
73390
|
if (!mime.startsWith("image/")) return null;
|
|
71701
73391
|
const auth = await resolveClaudeAuth(db);
|
|
71702
73392
|
if (!auth) return null;
|
|
71703
73393
|
try {
|
|
71704
|
-
const text = await describeImage(auth,
|
|
73394
|
+
const text = await describeImage(auth, path3);
|
|
71705
73395
|
return text.trim() ? { text, skip: false } : null;
|
|
71706
73396
|
} catch (e6) {
|
|
71707
73397
|
console.warn("[ingest] image vision failed:", e6.message);
|
|
71708
73398
|
return null;
|
|
71709
73399
|
}
|
|
71710
73400
|
}
|
|
71711
|
-
async function extractSource(db,
|
|
71712
|
-
const vision = await extractImage(db,
|
|
73401
|
+
async function extractSource(db, path3, mime, name) {
|
|
73402
|
+
const vision = await extractImage(db, path3, mime);
|
|
71713
73403
|
if (vision) return vision;
|
|
71714
|
-
const parsed = await parseFile(
|
|
73404
|
+
const parsed = await parseFile(path3, mime, name);
|
|
71715
73405
|
if (!parsed.skip) return parsed;
|
|
71716
73406
|
if (mime === "application/pdf") {
|
|
71717
73407
|
const auth = await resolveClaudeAuth(db);
|
|
71718
73408
|
if (auth) {
|
|
71719
73409
|
try {
|
|
71720
|
-
const text = await describePdf(auth,
|
|
73410
|
+
const text = await describePdf(auth, path3);
|
|
71721
73411
|
if (text.trim()) return { ...parsed, text, skip: false };
|
|
71722
73412
|
} catch (e6) {
|
|
71723
73413
|
console.warn("[ingest] Claude PDF read failed:", e6.message);
|
|
@@ -71730,7 +73420,6 @@ function looksLikeUrl(s2) {
|
|
|
71730
73420
|
const t8 = s2.trim();
|
|
71731
73421
|
return /^https?:\/\/\S+$/i.test(t8) && !/\s/.test(t8);
|
|
71732
73422
|
}
|
|
71733
|
-
var MAX_INGEST_BYTES = 5e7;
|
|
71734
73423
|
function readBuffer2(req, maxBytes = MAX_INGEST_BYTES) {
|
|
71735
73424
|
return new Promise((resolve_, reject) => {
|
|
71736
73425
|
const chunks = [];
|
|
@@ -71792,9 +73481,15 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71792
73481
|
const tmp = join28(tmpdir2(), `lattice-ingest-${crypto.randomUUID()}${extname2(name2)}`);
|
|
71793
73482
|
let result;
|
|
71794
73483
|
let blob = null;
|
|
73484
|
+
let autoImport = null;
|
|
71795
73485
|
try {
|
|
71796
73486
|
await writeFile2(tmp, buf);
|
|
71797
73487
|
result = await extractSource(ctx.db, tmp, mime2, name2);
|
|
73488
|
+
try {
|
|
73489
|
+
autoImport = await autoImportStructured(ctx.db, ctx.configPath ?? null, tmp, name2);
|
|
73490
|
+
} catch (e6) {
|
|
73491
|
+
console.warn("[ingest] auto-import skipped:", e6.message);
|
|
73492
|
+
}
|
|
71798
73493
|
if (ctx.latticeRoot && !realPath && shouldRetainUploadBlob(mime2, name2)) {
|
|
71799
73494
|
try {
|
|
71800
73495
|
const meta = await attachBlob(tmp, ctx.latticeRoot);
|
|
@@ -71810,7 +73505,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71810
73505
|
let s3Status = null;
|
|
71811
73506
|
const s3cfg = resolveActiveS3Config(ctx.configPath);
|
|
71812
73507
|
if (s3cfg) {
|
|
71813
|
-
const sha256 = blob?.sha256 ??
|
|
73508
|
+
const sha256 = blob?.sha256 ?? createHash12("sha256").update(buf).digest("hex");
|
|
71814
73509
|
const key = s3Key(s3cfg.prefix, sha256);
|
|
71815
73510
|
try {
|
|
71816
73511
|
const store = await createS3Store(s3cfg);
|
|
@@ -71833,7 +73528,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71833
73528
|
}
|
|
71834
73529
|
}
|
|
71835
73530
|
const fileId = crypto.randomUUID();
|
|
71836
|
-
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ??
|
|
73531
|
+
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ?? createHash12("sha256").update(buf).digest("hex");
|
|
71837
73532
|
const uploadRow = {
|
|
71838
73533
|
id: fileId,
|
|
71839
73534
|
...fileIdentity(name2, fileId),
|
|
@@ -71871,6 +73566,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71871
73566
|
},
|
|
71872
73567
|
forcePrivate2 ? "private" : void 0
|
|
71873
73568
|
);
|
|
73569
|
+
if (autoImport?.reason) autoImport.fileId = id2;
|
|
71874
73570
|
try {
|
|
71875
73571
|
const dedupCtx = {
|
|
71876
73572
|
db: ctx.db,
|
|
@@ -71898,6 +73594,15 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71898
73594
|
e6 instanceof Error ? e6.message : String(e6)
|
|
71899
73595
|
);
|
|
71900
73596
|
}
|
|
73597
|
+
if (autoImport?.imported) {
|
|
73598
|
+
ctx.feed.publish({
|
|
73599
|
+
table: autoImport.tables[0] ?? "files",
|
|
73600
|
+
op: "insert",
|
|
73601
|
+
rowId: null,
|
|
73602
|
+
source: "system",
|
|
73603
|
+
summary: `Imported the ${autoImport.asOf ?? ""} snapshot of "${name2}" \u2014 ${String(autoImport.rows)} rows across ${String(autoImport.tables.length)} tables`
|
|
73604
|
+
});
|
|
73605
|
+
}
|
|
71901
73606
|
let suggestedLinks = [];
|
|
71902
73607
|
if (!result.skip) {
|
|
71903
73608
|
const links = await enrichOrFail(mctx, ctx.db, id2, result.text, name2, ctx, res, forcePrivate2);
|
|
@@ -71910,6 +73615,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71910
73615
|
id: id2,
|
|
71911
73616
|
extraction_status: result.skip ? "skipped" : "extracted",
|
|
71912
73617
|
suggestedLinks,
|
|
73618
|
+
...autoImport ? { autoImport } : {},
|
|
71913
73619
|
// Present only when S3 is enabled for this workspace. 'failed' tells the
|
|
71914
73620
|
// uploader the bytes did NOT reach the shared bucket — other members would
|
|
71915
73621
|
// 404 until it's re-uploaded — so the GUI can warn rather than imply a
|
|
@@ -71995,7 +73701,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
71995
73701
|
sendJson(res, { error: "path is required" }, 400);
|
|
71996
73702
|
return true;
|
|
71997
73703
|
}
|
|
71998
|
-
const abs =
|
|
73704
|
+
const abs = resolve11(rawPath);
|
|
71999
73705
|
let size = 0;
|
|
72000
73706
|
try {
|
|
72001
73707
|
const st = statSync8(abs);
|
|
@@ -72012,7 +73718,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
72012
73718
|
sendJson(res, { error: "file too large" }, 413);
|
|
72013
73719
|
return true;
|
|
72014
73720
|
}
|
|
72015
|
-
const name =
|
|
73721
|
+
const name = basename11(abs);
|
|
72016
73722
|
const mime = mimeFor(name);
|
|
72017
73723
|
const localFileId = crypto.randomUUID();
|
|
72018
73724
|
const localRow = {
|
|
@@ -72078,6 +73784,146 @@ ${err.stack ?? ""}`
|
|
|
72078
73784
|
return true;
|
|
72079
73785
|
}
|
|
72080
73786
|
|
|
73787
|
+
// src/gui/import-routes.ts
|
|
73788
|
+
init_adapter();
|
|
73789
|
+
init_http();
|
|
73790
|
+
import { existsSync as existsSync27, readFileSync as readFileSync23, statSync as statSync9 } from "fs";
|
|
73791
|
+
import { isAbsolute as isAbsolute4, join as join29 } from "path";
|
|
73792
|
+
init_native_entities();
|
|
73793
|
+
function badRequest(message) {
|
|
73794
|
+
const e6 = new Error(message);
|
|
73795
|
+
e6.statusCode = 400;
|
|
73796
|
+
return e6;
|
|
73797
|
+
}
|
|
73798
|
+
function localPathOf2(row, latticeRoot) {
|
|
73799
|
+
if (row.ref_kind === "local_ref" && row.ref_uri) return row.ref_uri;
|
|
73800
|
+
if ((row.ref_kind === "blob" || row.ref_kind === "cloud_ref") && row.blob_path) {
|
|
73801
|
+
return isAbsolute4(row.blob_path) ? row.blob_path : latticeRoot ? join29(latticeRoot, row.blob_path) : null;
|
|
73802
|
+
}
|
|
73803
|
+
return null;
|
|
73804
|
+
}
|
|
73805
|
+
function existingDataTables2(db) {
|
|
73806
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
73807
|
+
const out = [];
|
|
73808
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
73809
|
+
if (native.has(t8)) continue;
|
|
73810
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
73811
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
73812
|
+
}
|
|
73813
|
+
return out;
|
|
73814
|
+
}
|
|
73815
|
+
async function readImportSourceFromFile(db, fileId, latticeRoot) {
|
|
73816
|
+
const row = await getAsyncOrSync(
|
|
73817
|
+
db.adapter,
|
|
73818
|
+
`SELECT "id","original_name","mime","ref_kind","ref_uri","blob_path"
|
|
73819
|
+
FROM "files" WHERE "id" = ? AND "deleted_at" IS NULL LIMIT 1`,
|
|
73820
|
+
[fileId]
|
|
73821
|
+
);
|
|
73822
|
+
if (!row) throw badRequest("Unknown import file: " + fileId);
|
|
73823
|
+
const path3 = localPathOf2(row, latticeRoot);
|
|
73824
|
+
if (!path3 || !existsSync27(path3)) {
|
|
73825
|
+
throw badRequest("The import file\u2019s bytes are not available locally.");
|
|
73826
|
+
}
|
|
73827
|
+
const sizeBytes = statSync9(path3).size;
|
|
73828
|
+
if (sizeBytes > MAX_INGEST_BYTES) {
|
|
73829
|
+
throw badRequest(
|
|
73830
|
+
`The import file is too large (${String(Math.round(sizeBytes / 1e6))} MB); the limit is ${String(Math.round(MAX_INGEST_BYTES / 1e6))} MB.`
|
|
73831
|
+
);
|
|
73832
|
+
}
|
|
73833
|
+
const name = row.original_name ?? "";
|
|
73834
|
+
const mime = row.mime ?? "";
|
|
73835
|
+
if (/\.xlsx?$/i.test(name) || mime.includes("spreadsheet") || mime.includes("excel")) {
|
|
73836
|
+
return excelToRecords(path3);
|
|
73837
|
+
}
|
|
73838
|
+
let parsed;
|
|
73839
|
+
try {
|
|
73840
|
+
parsed = JSON.parse(readFileSync23(path3, "utf8"));
|
|
73841
|
+
} catch {
|
|
73842
|
+
throw badRequest("The import file is not valid JSON.");
|
|
73843
|
+
}
|
|
73844
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
73845
|
+
throw badRequest("Expected a JSON object whose keys are record arrays.");
|
|
73846
|
+
}
|
|
73847
|
+
return parsed;
|
|
73848
|
+
}
|
|
73849
|
+
async function dispatchImportRoute(req, res, deps) {
|
|
73850
|
+
const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
|
|
73851
|
+
if (req.method !== "POST" || pathname !== "/api/import/apply") return false;
|
|
73852
|
+
const body = await readJson(req).catch(() => ({}));
|
|
73853
|
+
const fileId = typeof body.fileId === "string" ? body.fileId : "";
|
|
73854
|
+
const mode = body.mode === "schema" || body.mode === "contents" ? body.mode : "both";
|
|
73855
|
+
const asOf = typeof body.asOf === "string" && /^\d{4}-\d{2}-\d{2}$/.test(body.asOf.trim()) ? body.asOf.trim() : null;
|
|
73856
|
+
const asOfColumn = typeof body.asOfColumn === "string" && body.asOfColumn.trim() ? body.asOfColumn.trim() : null;
|
|
73857
|
+
if (!fileId) {
|
|
73858
|
+
sendJson(res, { error: "fileId is required" }, 400);
|
|
73859
|
+
return true;
|
|
73860
|
+
}
|
|
73861
|
+
res.writeHead(200, {
|
|
73862
|
+
"content-type": "application/x-ndjson; charset=utf-8",
|
|
73863
|
+
"cache-control": "no-store"
|
|
73864
|
+
});
|
|
73865
|
+
const emit = (p3) => {
|
|
73866
|
+
res.write(JSON.stringify(p3) + "\n");
|
|
73867
|
+
};
|
|
73868
|
+
try {
|
|
73869
|
+
emit({ phase: "parse", message: "Reading source\u2026" });
|
|
73870
|
+
const data = await readImportSourceFromFile(deps.db, fileId, deps.latticeRoot);
|
|
73871
|
+
emit({ phase: "infer", message: "Analyzing schema\u2026" });
|
|
73872
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
73873
|
+
inferSchema(data),
|
|
73874
|
+
data
|
|
73875
|
+
);
|
|
73876
|
+
emit({
|
|
73877
|
+
phase: "infer",
|
|
73878
|
+
message: `Found ${String(inferredPlan.entities.length)} entities, ${String(inferredPlan.dimensions.length)} dimensions, ${String(inferredPlan.linkages.length)} links`
|
|
73879
|
+
});
|
|
73880
|
+
const match = matchSchemaToExisting(existingDataTables2(deps.db), inferredPlan);
|
|
73881
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, match.rename);
|
|
73882
|
+
if (views.length > 0) {
|
|
73883
|
+
emit({
|
|
73884
|
+
phase: "detect",
|
|
73885
|
+
message: `Detected ${String(views.length)} reconstructable views (no duplicated rows)`
|
|
73886
|
+
});
|
|
73887
|
+
}
|
|
73888
|
+
if (match.isKnownDocument) {
|
|
73889
|
+
emit({
|
|
73890
|
+
phase: "detect",
|
|
73891
|
+
message: `Recognized as a new period of an existing document \u2014 ${String(match.matchedCount)} of ${String(match.totalEntities)} tables matched`
|
|
73892
|
+
});
|
|
73893
|
+
}
|
|
73894
|
+
if (asOfColumn) {
|
|
73895
|
+
emit({ phase: "infer", message: `Dating each row by its "${asOfColumn}" column` });
|
|
73896
|
+
} else if (asOf) {
|
|
73897
|
+
emit({ phase: "infer", message: `Importing as a snapshot dated ${asOf}` });
|
|
73898
|
+
}
|
|
73899
|
+
const result = await materializeImport(
|
|
73900
|
+
{ db: deps.db, configPath: deps.configPath },
|
|
73901
|
+
data,
|
|
73902
|
+
plan,
|
|
73903
|
+
views,
|
|
73904
|
+
{
|
|
73905
|
+
mode,
|
|
73906
|
+
asOf,
|
|
73907
|
+
asOfColumn,
|
|
73908
|
+
onProgress: async (p3) => {
|
|
73909
|
+
emit({ ...p3 });
|
|
73910
|
+
await new Promise((r6) => setImmediate(r6));
|
|
73911
|
+
}
|
|
73912
|
+
}
|
|
73913
|
+
);
|
|
73914
|
+
for (const t8 of result.tablesCreated) {
|
|
73915
|
+
deps.validTables.add(t8);
|
|
73916
|
+
const cols = deps.db.getRegisteredColumns(t8);
|
|
73917
|
+
if (cols && "deleted_at" in cols) deps.softDeletable.add(t8);
|
|
73918
|
+
}
|
|
73919
|
+
emit({ phase: "done", ok: true, result });
|
|
73920
|
+
} catch (e6) {
|
|
73921
|
+
emit({ phase: "error", message: e6.message });
|
|
73922
|
+
}
|
|
73923
|
+
res.end();
|
|
73924
|
+
return true;
|
|
73925
|
+
}
|
|
73926
|
+
|
|
72081
73927
|
// src/gui/read-routes.ts
|
|
72082
73928
|
init_http();
|
|
72083
73929
|
init_data();
|
|
@@ -72366,7 +74212,13 @@ async function handleReadRoutes(req, res, ctx, deps) {
|
|
|
72366
74212
|
return true;
|
|
72367
74213
|
}
|
|
72368
74214
|
if (method === "GET" && pathname === "/api/history") {
|
|
72369
|
-
const
|
|
74215
|
+
const limitRaw = url.searchParams.get("limit");
|
|
74216
|
+
const parsedLimit = parsePageParam(limitRaw, "limit");
|
|
74217
|
+
if (parsedLimit === "invalid") {
|
|
74218
|
+
sendJson(res, { error: "limit must be a non-negative integer" }, 400);
|
|
74219
|
+
return true;
|
|
74220
|
+
}
|
|
74221
|
+
const limit = limitRaw === null ? 200 : parsedLimit;
|
|
72370
74222
|
const filterTable = url.searchParams.get("table");
|
|
72371
74223
|
const raw = await active.db.query("_lattice_gui_audit", { limit });
|
|
72372
74224
|
let entries = raw.map(parseAudit).sort((a6, b6) => b6.ts.localeCompare(a6.ts));
|
|
@@ -73410,8 +75262,8 @@ async function handleHistoryRoutes(req, res, ctx, deps) {
|
|
|
73410
75262
|
|
|
73411
75263
|
// src/gui/workspaces-routes.ts
|
|
73412
75264
|
init_http();
|
|
73413
|
-
import { resolve as
|
|
73414
|
-
import { existsSync as
|
|
75265
|
+
import { resolve as resolve12 } from "path";
|
|
75266
|
+
import { existsSync as existsSync28, rmSync } from "fs";
|
|
73415
75267
|
init_workspace();
|
|
73416
75268
|
init_lattice_root();
|
|
73417
75269
|
init_user_config();
|
|
@@ -73419,7 +75271,7 @@ function cleanupWorkspaceFiles(root6, ws) {
|
|
|
73419
75271
|
if (!ws.configPath && ws.kind === "local") {
|
|
73420
75272
|
rmSync(workspaceDir(root6, ws.dir), { recursive: true, force: true });
|
|
73421
75273
|
} else if (ws.kind === "cloud") {
|
|
73422
|
-
if (ws.configPath &&
|
|
75274
|
+
if (ws.configPath && existsSync28(ws.configPath)) {
|
|
73423
75275
|
rmSync(ws.configPath, { force: true });
|
|
73424
75276
|
}
|
|
73425
75277
|
const labelMatch = /^\$\{LATTICE_DB:([A-Za-z0-9._-]+)\}$/.exec(ws.db.trim());
|
|
@@ -73573,7 +75425,7 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
73573
75425
|
return true;
|
|
73574
75426
|
}
|
|
73575
75427
|
const wsPaths = resolveWorkspacePaths(latticeRoot, ws);
|
|
73576
|
-
const isActive =
|
|
75428
|
+
const isActive = resolve12(active.configPath) === resolve12(wsPaths.configPath);
|
|
73577
75429
|
let switchedTo = null;
|
|
73578
75430
|
if (isActive) {
|
|
73579
75431
|
const fallback = listWorkspaces(latticeRoot).find((w2) => w2.id !== ws.id);
|
|
@@ -73622,40 +75474,40 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
73622
75474
|
|
|
73623
75475
|
// src/gui/databases-routes.ts
|
|
73624
75476
|
init_http();
|
|
73625
|
-
import { basename as
|
|
73626
|
-
import { existsSync as
|
|
75477
|
+
import { basename as basename13, resolve as resolve14 } from "path";
|
|
75478
|
+
import { existsSync as existsSync30 } from "fs";
|
|
73627
75479
|
init_parser();
|
|
73628
75480
|
|
|
73629
75481
|
// src/gui/config-paths.ts
|
|
73630
75482
|
init_parser();
|
|
73631
|
-
import { basename as
|
|
75483
|
+
import { basename as basename12, dirname as dirname16, join as join30, resolve as resolve13 } from "path";
|
|
73632
75484
|
import {
|
|
73633
|
-
existsSync as
|
|
75485
|
+
existsSync as existsSync29,
|
|
73634
75486
|
mkdirSync as mkdirSync12,
|
|
73635
|
-
readFileSync as
|
|
75487
|
+
readFileSync as readFileSync24,
|
|
73636
75488
|
readdirSync as readdirSync8,
|
|
73637
75489
|
unlinkSync as unlinkSync5,
|
|
73638
75490
|
writeFileSync as writeFileSync10
|
|
73639
75491
|
} from "fs";
|
|
73640
75492
|
import { parseDocument as parseDocument7 } from "yaml";
|
|
73641
75493
|
function resolveOutputDirForConfig(configPath) {
|
|
73642
|
-
const base = dirname16(
|
|
75494
|
+
const base = dirname16(resolve13(configPath));
|
|
73643
75495
|
for (const dir of ["context", ".", "generated"]) {
|
|
73644
|
-
const abs =
|
|
73645
|
-
if (
|
|
75496
|
+
const abs = resolve13(base, dir);
|
|
75497
|
+
if (existsSync29(join30(abs, ".lattice", "manifest.json"))) return abs;
|
|
73646
75498
|
}
|
|
73647
|
-
return
|
|
75499
|
+
return resolve13(base, "context");
|
|
73648
75500
|
}
|
|
73649
75501
|
function friendlyConfigName(parsedName, configPath) {
|
|
73650
75502
|
if (parsedName && parsedName.trim().length > 0) return parsedName.trim();
|
|
73651
|
-
return
|
|
75503
|
+
return basename12(configPath).replace(/\.(ya?ml)$/, "");
|
|
73652
75504
|
}
|
|
73653
75505
|
function listConfigs(activeConfigPath) {
|
|
73654
75506
|
const dir = dirname16(activeConfigPath);
|
|
73655
75507
|
const entries = [];
|
|
73656
75508
|
for (const fname of readdirSync8(dir)) {
|
|
73657
75509
|
if (!fname.endsWith(".yml") && !fname.endsWith(".yaml")) continue;
|
|
73658
|
-
const full =
|
|
75510
|
+
const full = join30(dir, fname);
|
|
73659
75511
|
try {
|
|
73660
75512
|
const parsed = parseConfigFile(full);
|
|
73661
75513
|
entries.push({
|
|
@@ -73666,7 +75518,7 @@ function listConfigs(activeConfigPath) {
|
|
|
73666
75518
|
// `label` is the friendly DB name — what the user sees in the
|
|
73667
75519
|
// dropdown + settings. Falls back to the basename when unset.
|
|
73668
75520
|
label: friendlyConfigName(parsed.name, full),
|
|
73669
|
-
dbFile:
|
|
75521
|
+
dbFile: basename12(parsed.dbPath),
|
|
73670
75522
|
active: full === activeConfigPath,
|
|
73671
75523
|
// `${LATTICE_DB:...}` and postgres:// configs resolve to a
|
|
73672
75524
|
// postgres URL; everything else is a local SQLite file. This
|
|
@@ -73683,37 +75535,37 @@ function createBlankConfig(activeConfigPath, dbName) {
|
|
|
73683
75535
|
const dir = dirname16(activeConfigPath);
|
|
73684
75536
|
const slug = dbName.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
73685
75537
|
if (!slug) throw new Error("Workspace name must contain at least one alphanumeric character");
|
|
73686
|
-
const configPath =
|
|
73687
|
-
if (
|
|
75538
|
+
const configPath = join30(dir, `${slug}.config.yml`);
|
|
75539
|
+
if (existsSync29(configPath)) throw new Error(`Config already exists: ${slug}.config.yml`);
|
|
73688
75540
|
const yaml = `db: ./data/${slug}.db
|
|
73689
75541
|
|
|
73690
75542
|
entities: {}
|
|
73691
75543
|
`;
|
|
73692
75544
|
writeFileSync10(configPath, yaml, "utf8");
|
|
73693
|
-
mkdirSync12(
|
|
75545
|
+
mkdirSync12(join30(dir, "data"), { recursive: true });
|
|
73694
75546
|
return configPath;
|
|
73695
75547
|
}
|
|
73696
75548
|
function sqliteFileForConfig(configPath) {
|
|
73697
|
-
const dbVal = parseDocument7(
|
|
75549
|
+
const dbVal = parseDocument7(readFileSync24(configPath, "utf8")).get("db");
|
|
73698
75550
|
const raw = (typeof dbVal === "string" ? dbVal : "").trim();
|
|
73699
75551
|
if (!raw) return null;
|
|
73700
75552
|
if (isPostgresUrl(raw) || raw.startsWith("${LATTICE_DB:")) return null;
|
|
73701
75553
|
if (raw === ":memory:" || raw.startsWith("file:")) return null;
|
|
73702
|
-
return
|
|
75554
|
+
return resolve13(dirname16(configPath), raw);
|
|
73703
75555
|
}
|
|
73704
75556
|
function deleteDatabaseFiles(targetConfigPath) {
|
|
73705
75557
|
const sqliteFile = sqliteFileForConfig(targetConfigPath);
|
|
73706
75558
|
unlinkSync5(targetConfigPath);
|
|
73707
75559
|
let deletedDbFile = null;
|
|
73708
|
-
if (sqliteFile &&
|
|
75560
|
+
if (sqliteFile && existsSync29(sqliteFile)) {
|
|
73709
75561
|
unlinkSync5(sqliteFile);
|
|
73710
75562
|
deletedDbFile = sqliteFile;
|
|
73711
75563
|
for (const suffix of ["-wal", "-shm", "-journal"]) {
|
|
73712
75564
|
const sidecar = sqliteFile + suffix;
|
|
73713
|
-
if (
|
|
75565
|
+
if (existsSync29(sidecar)) unlinkSync5(sidecar);
|
|
73714
75566
|
}
|
|
73715
75567
|
}
|
|
73716
|
-
return { deletedConfig:
|
|
75568
|
+
return { deletedConfig: basename12(targetConfigPath), deletedDbFile };
|
|
73717
75569
|
}
|
|
73718
75570
|
|
|
73719
75571
|
// src/gui/databases-routes.ts
|
|
@@ -73729,7 +75581,7 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
73729
75581
|
sendJson(res, {
|
|
73730
75582
|
current: {
|
|
73731
75583
|
path: active.configPath,
|
|
73732
|
-
dbFile:
|
|
75584
|
+
dbFile: basename13(parsedActive.dbPath),
|
|
73733
75585
|
label: friendlyLabel,
|
|
73734
75586
|
kind
|
|
73735
75587
|
},
|
|
@@ -73743,8 +75595,8 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
73743
75595
|
sendJson(res, { error: "path must be a string" }, 400);
|
|
73744
75596
|
return true;
|
|
73745
75597
|
}
|
|
73746
|
-
const newPath =
|
|
73747
|
-
if (!
|
|
75598
|
+
const newPath = resolve14(body.path);
|
|
75599
|
+
if (!existsSync30(newPath)) {
|
|
73748
75600
|
sendJson(res, { error: `Config not found: ${newPath}` }, 400);
|
|
73749
75601
|
return true;
|
|
73750
75602
|
}
|
|
@@ -73786,16 +75638,16 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
73786
75638
|
sendJson(res, { error: "path must be a non-empty string" }, 400);
|
|
73787
75639
|
return true;
|
|
73788
75640
|
}
|
|
73789
|
-
const target =
|
|
75641
|
+
const target = resolve14(body.path);
|
|
73790
75642
|
const known = listConfigs(active.configPath);
|
|
73791
|
-
const match = known.find((c6) =>
|
|
75643
|
+
const match = known.find((c6) => resolve14(c6.path) === target);
|
|
73792
75644
|
if (!match) {
|
|
73793
75645
|
sendJson(res, { error: `Not a known database config: ${target}` }, 400);
|
|
73794
75646
|
return true;
|
|
73795
75647
|
}
|
|
73796
75648
|
let switchedTo = null;
|
|
73797
|
-
if (
|
|
73798
|
-
const fallback = known.find((c6) =>
|
|
75649
|
+
if (resolve14(active.configPath) === target) {
|
|
75650
|
+
const fallback = known.find((c6) => resolve14(c6.path) !== target);
|
|
73799
75651
|
if (!fallback) {
|
|
73800
75652
|
sendJson(
|
|
73801
75653
|
res,
|
|
@@ -73886,20 +75738,26 @@ async function listenWithPortFallback(server, startPort, host) {
|
|
|
73886
75738
|
throw new Error(`No available port found starting at ${String(startPort)}`);
|
|
73887
75739
|
}
|
|
73888
75740
|
async function startGuiServer(options) {
|
|
73889
|
-
const bootConfigPath = options.configPath ?
|
|
73890
|
-
const bootOutputDir = options.outputDir ?
|
|
75741
|
+
const bootConfigPath = options.configPath ? resolve15(options.configPath) : null;
|
|
75742
|
+
const bootOutputDir = options.outputDir ? resolve15(options.outputDir) : null;
|
|
73891
75743
|
const startPort = options.port ?? 4317;
|
|
73892
75744
|
const host = options.host ?? "127.0.0.1";
|
|
75745
|
+
const isLoopbackHost2 = host === "localhost" || host === "::1" || host.startsWith("127.");
|
|
75746
|
+
if (!isLoopbackHost2) {
|
|
75747
|
+
console.warn(
|
|
75748
|
+
`[lattice] GUI is binding to a non-loopback address (${host}); its data routes are UNAUTHENTICATED and will be reachable from the network.`
|
|
75749
|
+
);
|
|
75750
|
+
}
|
|
73893
75751
|
const autoRender = options.autoRender ?? false;
|
|
73894
75752
|
const guiVersion = options.version ?? "";
|
|
73895
75753
|
const sessionId = crypto.randomUUID();
|
|
73896
75754
|
let updateService = null;
|
|
73897
75755
|
let activeRef = bootConfigPath && bootOutputDir ? await openConfig(bootConfigPath, bootOutputDir, autoRender, options.realtimeWatchdogMs) : null;
|
|
73898
|
-
const latticeRoot = (bootConfigPath ? findLatticeRoot(dirname17(bootConfigPath)) : null) ?? (options.latticeRoot ?
|
|
75756
|
+
const latticeRoot = (bootConfigPath ? findLatticeRoot(dirname17(bootConfigPath)) : null) ?? (options.latticeRoot ? resolve15(options.latticeRoot) : null);
|
|
73899
75757
|
let currentWorkspaceId = null;
|
|
73900
75758
|
if (latticeRoot && bootConfigPath) {
|
|
73901
75759
|
const launched = listWorkspaces(latticeRoot).find(
|
|
73902
|
-
(w2) =>
|
|
75760
|
+
(w2) => resolve15(resolveWorkspacePaths(latticeRoot, w2).configPath) === resolve15(bootConfigPath)
|
|
73903
75761
|
);
|
|
73904
75762
|
if (launched) {
|
|
73905
75763
|
currentWorkspaceId = launched.id;
|
|
@@ -74244,6 +76102,22 @@ async function startGuiServer(options) {
|
|
|
74244
76102
|
});
|
|
74245
76103
|
}
|
|
74246
76104
|
},
|
|
76105
|
+
// ── Structured-source import (apply) ──
|
|
76106
|
+
// The importer is reachable only via dropping a file in the assistant
|
|
76107
|
+
// chat; this materializes the user-confirmed proposal, re-reading the
|
|
76108
|
+
// file's bytes from its `fileId` (its retained blob).
|
|
76109
|
+
{
|
|
76110
|
+
handle: async (req2, res2) => {
|
|
76111
|
+
if (!pathname.startsWith("/api/import/")) return false;
|
|
76112
|
+
return await dispatchImportRoute(req2, res2, {
|
|
76113
|
+
db: active.db,
|
|
76114
|
+
configPath: active.configPath,
|
|
76115
|
+
latticeRoot: dirname17(active.configPath),
|
|
76116
|
+
validTables: active.validTables,
|
|
76117
|
+
softDeletable: active.softDeletable
|
|
76118
|
+
});
|
|
76119
|
+
}
|
|
76120
|
+
},
|
|
74247
76121
|
// ── Files: blob serving + open-in-finder ──
|
|
74248
76122
|
{
|
|
74249
76123
|
handle: async (req2, res2) => {
|
|
@@ -74434,6 +76308,7 @@ ${e6.stack ?? ""}`
|
|
|
74434
76308
|
server,
|
|
74435
76309
|
port,
|
|
74436
76310
|
url,
|
|
76311
|
+
whenConverged: () => activeRef?.converged ?? Promise.resolve(),
|
|
74437
76312
|
close: () => new Promise((resolveClose, reject) => {
|
|
74438
76313
|
updateService?.stop();
|
|
74439
76314
|
for (const client of wss.clients) {
|
|
@@ -74742,10 +76617,10 @@ function printHelp() {
|
|
|
74742
76617
|
);
|
|
74743
76618
|
}
|
|
74744
76619
|
function getVersion() {
|
|
74745
|
-
if (true) return "4.1
|
|
76620
|
+
if (true) return "4.2.1";
|
|
74746
76621
|
try {
|
|
74747
76622
|
const pkgPath = new URL("../package.json", import.meta.url).pathname;
|
|
74748
|
-
const pkg = JSON.parse(
|
|
76623
|
+
const pkg = JSON.parse(readFileSync25(pkgPath, "utf-8"));
|
|
74749
76624
|
return pkg.version;
|
|
74750
76625
|
} catch {
|
|
74751
76626
|
return "unknown";
|
|
@@ -74776,10 +76651,10 @@ async function runUpdate() {
|
|
|
74776
76651
|
}
|
|
74777
76652
|
}
|
|
74778
76653
|
function runGenerate(args) {
|
|
74779
|
-
const configPath =
|
|
76654
|
+
const configPath = resolve16(args.config);
|
|
74780
76655
|
let raw;
|
|
74781
76656
|
try {
|
|
74782
|
-
raw =
|
|
76657
|
+
raw = readFileSync25(configPath, "utf-8");
|
|
74783
76658
|
} catch {
|
|
74784
76659
|
console.error(`Error: cannot read config file at "${configPath}"`);
|
|
74785
76660
|
process.exit(1);
|
|
@@ -74796,7 +76671,7 @@ function runGenerate(args) {
|
|
|
74796
76671
|
process.exit(1);
|
|
74797
76672
|
}
|
|
74798
76673
|
const configDir2 = dirname18(configPath);
|
|
74799
|
-
const outDir =
|
|
76674
|
+
const outDir = resolve16(args.out);
|
|
74800
76675
|
try {
|
|
74801
76676
|
const result = generateAll({ config, configDir: configDir2, outDir, scaffold: args.scaffold });
|
|
74802
76677
|
console.log(`Generated ${String(result.filesWritten.length)} file(s):`);
|
|
@@ -74809,8 +76684,8 @@ function runGenerate(args) {
|
|
|
74809
76684
|
}
|
|
74810
76685
|
}
|
|
74811
76686
|
async function runRender(args) {
|
|
74812
|
-
const outputDir =
|
|
74813
|
-
const configPath =
|
|
76687
|
+
const outputDir = resolve16(args.output);
|
|
76688
|
+
const configPath = resolve16(args.config);
|
|
74814
76689
|
let parsed;
|
|
74815
76690
|
try {
|
|
74816
76691
|
parsed = parseConfigFile(configPath);
|
|
@@ -74838,7 +76713,7 @@ async function runRender(args) {
|
|
|
74838
76713
|
}
|
|
74839
76714
|
}
|
|
74840
76715
|
async function runDoctor(args) {
|
|
74841
|
-
const db = new Lattice({ config:
|
|
76716
|
+
const db = new Lattice({ config: resolve16(args.config) });
|
|
74842
76717
|
try {
|
|
74843
76718
|
await db.init();
|
|
74844
76719
|
const report = await db.diagnoseRetrieval();
|
|
@@ -74865,7 +76740,7 @@ async function runSearch(args) {
|
|
|
74865
76740
|
console.error("Error: --table <table> is required for search");
|
|
74866
76741
|
process.exit(1);
|
|
74867
76742
|
}
|
|
74868
|
-
const db = new Lattice({ config:
|
|
76743
|
+
const db = new Lattice({ config: resolve16(args.config) });
|
|
74869
76744
|
try {
|
|
74870
76745
|
await db.init();
|
|
74871
76746
|
const results = await db.hybridSearch(args.table, args.query, { topK: args.topK ?? 10 });
|
|
@@ -74897,8 +76772,8 @@ async function runSearch(args) {
|
|
|
74897
76772
|
}
|
|
74898
76773
|
}
|
|
74899
76774
|
async function runReconcile(args, isDryRun) {
|
|
74900
|
-
const outputDir =
|
|
74901
|
-
const db = new Lattice({ config:
|
|
76775
|
+
const outputDir = resolve16(args.output);
|
|
76776
|
+
const db = new Lattice({ config: resolve16(args.config) });
|
|
74902
76777
|
try {
|
|
74903
76778
|
await db.init();
|
|
74904
76779
|
const start = Date.now();
|
|
@@ -74957,8 +76832,8 @@ function formatTimestamp() {
|
|
|
74957
76832
|
return `${hh}:${mm}:${ss}`;
|
|
74958
76833
|
}
|
|
74959
76834
|
async function runWatch(args) {
|
|
74960
|
-
const outputDir =
|
|
74961
|
-
const db = new Lattice({ config:
|
|
76835
|
+
const outputDir = resolve16(args.output);
|
|
76836
|
+
const db = new Lattice({ config: resolve16(args.config) });
|
|
74962
76837
|
try {
|
|
74963
76838
|
await db.init();
|
|
74964
76839
|
} catch (e6) {
|
|
@@ -75027,7 +76902,7 @@ async function runGui(args) {
|
|
|
75027
76902
|
if (args.root) process.env.LATTICE_ROOT = args.root;
|
|
75028
76903
|
const boot = ensureRootForGui({
|
|
75029
76904
|
startDir: args.root ?? process.cwd(),
|
|
75030
|
-
configPath:
|
|
76905
|
+
configPath: resolve16(args.config),
|
|
75031
76906
|
explicitConfig: args.config !== "./lattice.config.yml"
|
|
75032
76907
|
});
|
|
75033
76908
|
console.log(
|