latticesql 4.1.0 → 4.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/cli.js +2184 -309
- package/dist/index.cjs +2478 -571
- package/dist/index.d.cts +360 -1
- package/dist/index.d.ts +360 -1
- package/dist/index.js +2208 -316
- package/docs/api-reference.md +60 -4
- package/docs/architecture.md +24 -0
- package/docs/assistant.md +23 -0
- package/docs/configuration.md +27 -0
- package/docs/examples/dashboard.html +284 -0
- package/docs/importing.md +118 -0
- package/docs/retrieval.md +31 -0
- package/package.json +7 -3
package/dist/index.js
CHANGED
|
@@ -224,10 +224,10 @@ function manifestPath(outputDir) {
|
|
|
224
224
|
return join2(outputDir, ".lattice", "manifest.json");
|
|
225
225
|
}
|
|
226
226
|
function readManifest(outputDir) {
|
|
227
|
-
const
|
|
228
|
-
if (!existsSync2(
|
|
227
|
+
const path3 = manifestPath(outputDir);
|
|
228
|
+
if (!existsSync2(path3)) return null;
|
|
229
229
|
try {
|
|
230
|
-
return JSON.parse(readFileSync2(
|
|
230
|
+
return JSON.parse(readFileSync2(path3, "utf8"));
|
|
231
231
|
} catch {
|
|
232
232
|
return null;
|
|
233
233
|
}
|
|
@@ -415,20 +415,130 @@ var init_render_cursor = __esm({
|
|
|
415
415
|
}
|
|
416
416
|
});
|
|
417
417
|
|
|
418
|
+
// src/db/load-sqlite.ts
|
|
419
|
+
import path from "path";
|
|
420
|
+
import { createRequire } from "module";
|
|
421
|
+
import { spawnSync } from "child_process";
|
|
422
|
+
function runtimeRequire() {
|
|
423
|
+
const importMetaUrl = import.meta.url;
|
|
424
|
+
return importMetaUrl ? createRequire(importMetaUrl) : (
|
|
425
|
+
// CJS fallback — Node provides `require` on every CJS module scope. Under
|
|
426
|
+
// tsup's CJS output `import.meta.url` is rewritten to undefined, so this
|
|
427
|
+
// branch keeps the loader working in the published .cjs bundle.
|
|
428
|
+
__require
|
|
429
|
+
);
|
|
430
|
+
}
|
|
431
|
+
function asCtor(mod) {
|
|
432
|
+
return mod.default ?? mod;
|
|
433
|
+
}
|
|
434
|
+
function isAbiMismatch(err) {
|
|
435
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
436
|
+
const code = err.code;
|
|
437
|
+
return message.includes("NODE_MODULE_VERSION") || code === "ERR_DLOPEN_FAILED" || message.includes("was compiled against a different Node.js version");
|
|
438
|
+
}
|
|
439
|
+
function autoRebuildDisabled() {
|
|
440
|
+
const v2 = process.env.LATTICE_SQLITE_NO_AUTOREBUILD;
|
|
441
|
+
if (!v2) return false;
|
|
442
|
+
const normalized = v2.trim().toLowerCase();
|
|
443
|
+
return normalized !== "" && normalized !== "0" && normalized !== "false" && normalized !== "no";
|
|
444
|
+
}
|
|
445
|
+
function installRootFor(req) {
|
|
446
|
+
const pkgJsonPath = req.resolve("better-sqlite3/package.json");
|
|
447
|
+
return path.resolve(path.dirname(pkgJsonPath), "..", "..");
|
|
448
|
+
}
|
|
449
|
+
function defaultRebuild(installRoot) {
|
|
450
|
+
const npmBin = process.platform === "win32" ? "npm.cmd" : "npm";
|
|
451
|
+
const res = spawnSync(npmBin, ["rebuild", "better-sqlite3"], {
|
|
452
|
+
cwd: installRoot,
|
|
453
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
454
|
+
encoding: "utf8",
|
|
455
|
+
timeout: 5 * 60 * 1e3
|
|
456
|
+
});
|
|
457
|
+
if (res.error) {
|
|
458
|
+
return { ok: false, reason: res.error.message };
|
|
459
|
+
}
|
|
460
|
+
if (res.status !== 0) {
|
|
461
|
+
const stderr = res.stderr.trim();
|
|
462
|
+
const tail = stderr ? stderr.slice(-300) : `npm rebuild exited with code ${String(res.status)}`;
|
|
463
|
+
return { ok: false, reason: tail };
|
|
464
|
+
}
|
|
465
|
+
return { ok: true };
|
|
466
|
+
}
|
|
467
|
+
function resolveSqliteCtor(options = {}) {
|
|
468
|
+
const req = options.require ?? runtimeRequire();
|
|
469
|
+
const rebuild = options.rebuild ?? defaultRebuild;
|
|
470
|
+
const resolveInstallRoot = options.installRoot ?? installRootFor;
|
|
471
|
+
const log = options.log ?? ((msg) => process.stderr.write(msg + "\n"));
|
|
472
|
+
let firstError;
|
|
473
|
+
try {
|
|
474
|
+
return asCtor(req("better-sqlite3"));
|
|
475
|
+
} catch (err) {
|
|
476
|
+
firstError = err;
|
|
477
|
+
}
|
|
478
|
+
if (!isAbiMismatch(firstError)) {
|
|
479
|
+
throw new Error(PEER_DEP_MISSING_MESSAGE);
|
|
480
|
+
}
|
|
481
|
+
if (autoRebuildDisabled()) {
|
|
482
|
+
throw new Error(
|
|
483
|
+
rebuildFailedMessage("automatic rebuild is disabled (LATTICE_SQLITE_NO_AUTOREBUILD)")
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
log("[latticesql] SQLite engine built for a different Node runtime \u2014 rebuilding better-sqlite3\u2026");
|
|
487
|
+
let installRoot;
|
|
488
|
+
try {
|
|
489
|
+
installRoot = resolveInstallRoot(req);
|
|
490
|
+
} catch (err) {
|
|
491
|
+
throw new Error(
|
|
492
|
+
rebuildFailedMessage(
|
|
493
|
+
"could not locate the better-sqlite3 install root (" + (err instanceof Error ? err.message : String(err)) + ")"
|
|
494
|
+
)
|
|
495
|
+
);
|
|
496
|
+
}
|
|
497
|
+
const outcome = rebuild(installRoot);
|
|
498
|
+
if (!outcome.ok) {
|
|
499
|
+
throw new Error(rebuildFailedMessage(outcome.reason));
|
|
500
|
+
}
|
|
501
|
+
try {
|
|
502
|
+
return asCtor(req("better-sqlite3"));
|
|
503
|
+
} catch (err) {
|
|
504
|
+
throw new Error(
|
|
505
|
+
rebuildFailedMessage(
|
|
506
|
+
"the rebuilt module still failed to load (" + (err instanceof Error ? err.message : String(err)) + ")"
|
|
507
|
+
)
|
|
508
|
+
);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
function rebuildFailedMessage(reason) {
|
|
512
|
+
return "latticesql: the better-sqlite3 native module doesn\u2019t match this Node runtime and an automatic rebuild did not complete (" + reason + "). Run `npm rebuild better-sqlite3` (or reinstall) and retry.";
|
|
513
|
+
}
|
|
514
|
+
function loadSqlite() {
|
|
515
|
+
if (_ctor) return _ctor;
|
|
516
|
+
_ctor = resolveSqliteCtor();
|
|
517
|
+
return _ctor;
|
|
518
|
+
}
|
|
519
|
+
var PEER_DEP_MISSING_MESSAGE, _ctor;
|
|
520
|
+
var init_load_sqlite = __esm({
|
|
521
|
+
"src/db/load-sqlite.ts"() {
|
|
522
|
+
"use strict";
|
|
523
|
+
PEER_DEP_MISSING_MESSAGE = "better-sqlite3 is a required peer dependency of latticesql \u2014 install it (npm install better-sqlite3).";
|
|
524
|
+
_ctor = null;
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
|
|
418
528
|
// src/db/sqlite.ts
|
|
419
|
-
import Database from "better-sqlite3";
|
|
420
529
|
var SQLiteAdapter;
|
|
421
530
|
var init_sqlite = __esm({
|
|
422
531
|
"src/db/sqlite.ts"() {
|
|
423
532
|
"use strict";
|
|
533
|
+
init_load_sqlite();
|
|
424
534
|
SQLiteAdapter = class {
|
|
425
535
|
dialect = "sqlite";
|
|
426
536
|
_db = null;
|
|
427
537
|
_path;
|
|
428
538
|
_wal;
|
|
429
539
|
_busyTimeout;
|
|
430
|
-
constructor(
|
|
431
|
-
this._path =
|
|
540
|
+
constructor(path3, options) {
|
|
541
|
+
this._path = path3;
|
|
432
542
|
this._wal = options?.wal ?? true;
|
|
433
543
|
this._busyTimeout = options?.busyTimeout ?? 5e3;
|
|
434
544
|
}
|
|
@@ -437,7 +547,8 @@ var init_sqlite = __esm({
|
|
|
437
547
|
return this._db;
|
|
438
548
|
}
|
|
439
549
|
open() {
|
|
440
|
-
|
|
550
|
+
const Ctor = loadSqlite();
|
|
551
|
+
this._db = new Ctor(this._path);
|
|
441
552
|
this._db.pragma(`busy_timeout = ${this._busyTimeout.toString()}`);
|
|
442
553
|
if (this._wal) {
|
|
443
554
|
this._db.pragma("journal_mode = WAL");
|
|
@@ -609,16 +720,16 @@ var init_sqlite = __esm({
|
|
|
609
720
|
});
|
|
610
721
|
|
|
611
722
|
// src/db/postgres.ts
|
|
612
|
-
import
|
|
723
|
+
import path2 from "path";
|
|
613
724
|
import { fileURLToPath } from "url";
|
|
614
|
-
import { createRequire } from "module";
|
|
725
|
+
import { createRequire as createRequire2 } from "module";
|
|
615
726
|
function moduleContext() {
|
|
616
727
|
if (_moduleContext) return _moduleContext;
|
|
617
728
|
const importMetaUrl = import.meta.url;
|
|
618
729
|
if (importMetaUrl) {
|
|
619
730
|
_moduleContext = {
|
|
620
|
-
dir:
|
|
621
|
-
require:
|
|
731
|
+
dir: path2.dirname(fileURLToPath(importMetaUrl)),
|
|
732
|
+
require: createRequire2(importMetaUrl)
|
|
622
733
|
};
|
|
623
734
|
} else {
|
|
624
735
|
_moduleContext = { dir: __dirname, require: __require };
|
|
@@ -2547,14 +2658,14 @@ var init_core = __esm({
|
|
|
2547
2658
|
columnRef(f6) {
|
|
2548
2659
|
const col = `"${ident(f6.col)}"`;
|
|
2549
2660
|
if (f6.jsonPath === void 0) return { sql: col, params: [] };
|
|
2550
|
-
const
|
|
2661
|
+
const path3 = Array.isArray(f6.jsonPath) ? f6.jsonPath : [f6.jsonPath];
|
|
2551
2662
|
const numeric = isNumericComparison(f6);
|
|
2552
2663
|
if (this.adapter.dialect === "postgres") {
|
|
2553
2664
|
const extract2 = `((${col})::jsonb #>> ?::text[])`;
|
|
2554
2665
|
const sql2 = numeric ? `(${extract2})::numeric` : extract2;
|
|
2555
|
-
return { sql: sql2, params: [
|
|
2666
|
+
return { sql: sql2, params: [path3] };
|
|
2556
2667
|
}
|
|
2557
|
-
const jsonpath = `$.${
|
|
2668
|
+
const jsonpath = `$.${path3.join(".")}`;
|
|
2558
2669
|
const extract = `json_extract(${col}, ?)`;
|
|
2559
2670
|
const sql = numeric ? `CAST(${extract} AS REAL)` : extract;
|
|
2560
2671
|
return { sql, params: [jsonpath] };
|
|
@@ -5189,8 +5300,8 @@ var init_pipeline = __esm({
|
|
|
5189
5300
|
|
|
5190
5301
|
// src/render/interpolate.ts
|
|
5191
5302
|
function interpolate(template, row) {
|
|
5192
|
-
return template.replace(/\{\{([^}]+)\}\}/g, (_,
|
|
5193
|
-
const parts =
|
|
5303
|
+
return template.replace(/\{\{([^}]+)\}\}/g, (_, path3) => {
|
|
5304
|
+
const parts = path3.trim().split(".");
|
|
5194
5305
|
let val = row;
|
|
5195
5306
|
for (const part of parts) {
|
|
5196
5307
|
if (val == null || typeof val !== "object") return "";
|
|
@@ -5456,10 +5567,10 @@ function getOrCreateMasterKey() {
|
|
|
5456
5567
|
}
|
|
5457
5568
|
function readIdentity() {
|
|
5458
5569
|
const dir = ensureConfigDir();
|
|
5459
|
-
const
|
|
5460
|
-
if (!existsSync9(
|
|
5570
|
+
const path3 = join9(dir, IDENTITY_FILENAME);
|
|
5571
|
+
if (!existsSync9(path3)) return { ...EMPTY_IDENTITY };
|
|
5461
5572
|
try {
|
|
5462
|
-
const parsed = JSON.parse(readFileSync6(
|
|
5573
|
+
const parsed = JSON.parse(readFileSync6(path3, "utf8"));
|
|
5463
5574
|
return {
|
|
5464
5575
|
display_name: typeof parsed.display_name === "string" ? parsed.display_name : "",
|
|
5465
5576
|
email: typeof parsed.email === "string" ? parsed.email : ""
|
|
@@ -5470,26 +5581,26 @@ function readIdentity() {
|
|
|
5470
5581
|
}
|
|
5471
5582
|
function writeIdentity(identity) {
|
|
5472
5583
|
const dir = ensureConfigDir();
|
|
5473
|
-
const
|
|
5584
|
+
const path3 = join9(dir, IDENTITY_FILENAME);
|
|
5474
5585
|
const body = JSON.stringify(
|
|
5475
5586
|
{ display_name: identity.display_name, email: identity.email },
|
|
5476
5587
|
null,
|
|
5477
5588
|
2
|
|
5478
5589
|
);
|
|
5479
|
-
writeFileSync2(
|
|
5590
|
+
writeFileSync2(path3, body + "\n", "utf8");
|
|
5480
5591
|
if (platform2() !== "win32") {
|
|
5481
5592
|
try {
|
|
5482
|
-
chmodSync2(
|
|
5593
|
+
chmodSync2(path3, 384);
|
|
5483
5594
|
} catch {
|
|
5484
5595
|
}
|
|
5485
5596
|
}
|
|
5486
5597
|
}
|
|
5487
5598
|
function readPreferences() {
|
|
5488
5599
|
const dir = ensureConfigDir();
|
|
5489
|
-
const
|
|
5490
|
-
if (!existsSync9(
|
|
5600
|
+
const path3 = join9(dir, PREFERENCES_FILENAME);
|
|
5601
|
+
if (!existsSync9(path3)) return { ...DEFAULT_PREFERENCES };
|
|
5491
5602
|
try {
|
|
5492
|
-
const parsed = JSON.parse(readFileSync6(
|
|
5603
|
+
const parsed = JSON.parse(readFileSync6(path3, "utf8"));
|
|
5493
5604
|
const agg = typeof parsed.aggressiveness === "number" ? parsed.aggressiveness : NaN;
|
|
5494
5605
|
return {
|
|
5495
5606
|
show_system_tables: typeof parsed.show_system_tables === "boolean" ? parsed.show_system_tables : DEFAULT_PREFERENCES.show_system_tables,
|
|
@@ -5503,7 +5614,7 @@ function readPreferences() {
|
|
|
5503
5614
|
}
|
|
5504
5615
|
function writePreferences(prefs) {
|
|
5505
5616
|
const dir = ensureConfigDir();
|
|
5506
|
-
const
|
|
5617
|
+
const path3 = join9(dir, PREFERENCES_FILENAME);
|
|
5507
5618
|
const body = JSON.stringify(
|
|
5508
5619
|
{
|
|
5509
5620
|
show_system_tables: prefs.show_system_tables,
|
|
@@ -5514,10 +5625,10 @@ function writePreferences(prefs) {
|
|
|
5514
5625
|
null,
|
|
5515
5626
|
2
|
|
5516
5627
|
);
|
|
5517
|
-
writeFileSync2(
|
|
5628
|
+
writeFileSync2(path3, body + "\n", "utf8");
|
|
5518
5629
|
if (platform2() !== "win32") {
|
|
5519
5630
|
try {
|
|
5520
|
-
chmodSync2(
|
|
5631
|
+
chmodSync2(path3, 384);
|
|
5521
5632
|
} catch {
|
|
5522
5633
|
}
|
|
5523
5634
|
}
|
|
@@ -5550,7 +5661,9 @@ function withCredentialLock(fn) {
|
|
|
5550
5661
|
fd = openSync2(lockPath, "wx");
|
|
5551
5662
|
break;
|
|
5552
5663
|
} catch (err) {
|
|
5553
|
-
|
|
5664
|
+
const code = err.code;
|
|
5665
|
+
const contended = code === "EEXIST" || process.platform === "win32" && (code === "EPERM" || code === "EACCES");
|
|
5666
|
+
if (!contended) throw err;
|
|
5554
5667
|
try {
|
|
5555
5668
|
if (Date.now() - statSync4(lockPath).mtimeMs > LOCK_STALE_MS) {
|
|
5556
5669
|
unlinkSync3(lockPath);
|
|
@@ -5579,8 +5692,8 @@ function withCredentialLock(fn) {
|
|
|
5579
5692
|
}
|
|
5580
5693
|
}
|
|
5581
5694
|
}
|
|
5582
|
-
function writeFileAtomic(
|
|
5583
|
-
const tmp = `${
|
|
5695
|
+
function writeFileAtomic(path3, data) {
|
|
5696
|
+
const tmp = `${path3}.${String(process.pid)}.${randomBytes4(4).toString("hex")}.tmp`;
|
|
5584
5697
|
writeFileSync2(tmp, data, "utf8");
|
|
5585
5698
|
if (platform2() !== "win32") {
|
|
5586
5699
|
try {
|
|
@@ -5588,7 +5701,7 @@ function writeFileAtomic(path2, data) {
|
|
|
5588
5701
|
} catch {
|
|
5589
5702
|
}
|
|
5590
5703
|
}
|
|
5591
|
-
renameSync2(tmp,
|
|
5704
|
+
renameSync2(tmp, path3);
|
|
5592
5705
|
}
|
|
5593
5706
|
function mutateCredentials(mutate) {
|
|
5594
5707
|
withCredentialLock(() => {
|
|
@@ -5599,11 +5712,11 @@ function mutateCredentials(mutate) {
|
|
|
5599
5712
|
}
|
|
5600
5713
|
function loadCredentials() {
|
|
5601
5714
|
const dir = ensureConfigDir();
|
|
5602
|
-
const
|
|
5603
|
-
if (!existsSync9(
|
|
5715
|
+
const path3 = join9(dir, DB_CREDENTIALS_FILENAME);
|
|
5716
|
+
if (!existsSync9(path3)) return {};
|
|
5604
5717
|
const key = deriveKey(getOrCreateMasterKey());
|
|
5605
5718
|
try {
|
|
5606
|
-
const ciphertext = readFileSync6(
|
|
5719
|
+
const ciphertext = readFileSync6(path3, "utf8").trim();
|
|
5607
5720
|
const plaintext = decrypt(ciphertext, key);
|
|
5608
5721
|
const parsed = JSON.parse(plaintext);
|
|
5609
5722
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
@@ -5618,10 +5731,10 @@ function loadCredentials() {
|
|
|
5618
5731
|
}
|
|
5619
5732
|
function saveCredentials(creds) {
|
|
5620
5733
|
const dir = ensureConfigDir();
|
|
5621
|
-
const
|
|
5734
|
+
const path3 = join9(dir, DB_CREDENTIALS_FILENAME);
|
|
5622
5735
|
const key = deriveKey(getOrCreateMasterKey());
|
|
5623
5736
|
const ciphertext = encrypt(JSON.stringify(creds), key);
|
|
5624
|
-
writeFileAtomic(
|
|
5737
|
+
writeFileAtomic(path3, ciphertext + "\n");
|
|
5625
5738
|
}
|
|
5626
5739
|
function listDbCredentials() {
|
|
5627
5740
|
return Object.keys(loadCredentials()).sort();
|
|
@@ -5669,11 +5782,11 @@ function healRawDbUrl(configPath) {
|
|
|
5669
5782
|
}
|
|
5670
5783
|
function loadS3Configs() {
|
|
5671
5784
|
const dir = ensureConfigDir();
|
|
5672
|
-
const
|
|
5673
|
-
if (!existsSync9(
|
|
5785
|
+
const path3 = join9(dir, S3_CONFIG_FILENAME);
|
|
5786
|
+
if (!existsSync9(path3)) return {};
|
|
5674
5787
|
const key = deriveKey(getOrCreateMasterKey());
|
|
5675
5788
|
try {
|
|
5676
|
-
const parsed = JSON.parse(decrypt(readFileSync6(
|
|
5789
|
+
const parsed = JSON.parse(decrypt(readFileSync6(path3, "utf8").trim(), key));
|
|
5677
5790
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
5678
5791
|
return parsed;
|
|
5679
5792
|
}
|
|
@@ -5687,12 +5800,12 @@ function loadS3Configs() {
|
|
|
5687
5800
|
}
|
|
5688
5801
|
function saveS3Configs(cfgs) {
|
|
5689
5802
|
const dir = ensureConfigDir();
|
|
5690
|
-
const
|
|
5803
|
+
const path3 = join9(dir, S3_CONFIG_FILENAME);
|
|
5691
5804
|
const key = deriveKey(getOrCreateMasterKey());
|
|
5692
|
-
writeFileSync2(
|
|
5805
|
+
writeFileSync2(path3, encrypt(JSON.stringify(cfgs), key) + "\n", "utf8");
|
|
5693
5806
|
if (platform2() !== "win32") {
|
|
5694
5807
|
try {
|
|
5695
|
-
chmodSync2(
|
|
5808
|
+
chmodSync2(path3, 384);
|
|
5696
5809
|
} catch {
|
|
5697
5810
|
}
|
|
5698
5811
|
}
|
|
@@ -5729,11 +5842,11 @@ function deleteDbCredential(label) {
|
|
|
5729
5842
|
}
|
|
5730
5843
|
function loadAssistantCredentials() {
|
|
5731
5844
|
const dir = ensureConfigDir();
|
|
5732
|
-
const
|
|
5733
|
-
if (!existsSync9(
|
|
5845
|
+
const path3 = join9(dir, ASSISTANT_CREDENTIALS_FILENAME);
|
|
5846
|
+
if (!existsSync9(path3)) return {};
|
|
5734
5847
|
const key = deriveKey(getOrCreateMasterKey());
|
|
5735
5848
|
try {
|
|
5736
|
-
const ciphertext = readFileSync6(
|
|
5849
|
+
const ciphertext = readFileSync6(path3, "utf8").trim();
|
|
5737
5850
|
const plaintext = decrypt(ciphertext, key);
|
|
5738
5851
|
const parsed = JSON.parse(plaintext);
|
|
5739
5852
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
@@ -5748,13 +5861,13 @@ function loadAssistantCredentials() {
|
|
|
5748
5861
|
}
|
|
5749
5862
|
function saveAssistantCredentials(creds) {
|
|
5750
5863
|
const dir = ensureConfigDir();
|
|
5751
|
-
const
|
|
5864
|
+
const path3 = join9(dir, ASSISTANT_CREDENTIALS_FILENAME);
|
|
5752
5865
|
const key = deriveKey(getOrCreateMasterKey());
|
|
5753
5866
|
const ciphertext = encrypt(JSON.stringify(creds), key);
|
|
5754
|
-
writeFileSync2(
|
|
5867
|
+
writeFileSync2(path3, ciphertext + "\n", "utf8");
|
|
5755
5868
|
if (platform2() !== "win32") {
|
|
5756
5869
|
try {
|
|
5757
|
-
chmodSync2(
|
|
5870
|
+
chmodSync2(path3, 384);
|
|
5758
5871
|
} catch {
|
|
5759
5872
|
}
|
|
5760
5873
|
}
|
|
@@ -5815,25 +5928,25 @@ function listTokens() {
|
|
|
5815
5928
|
}
|
|
5816
5929
|
function readToken(label) {
|
|
5817
5930
|
assertSafeLabel(label);
|
|
5818
|
-
const
|
|
5819
|
-
if (!existsSync9(
|
|
5820
|
-
return readFileSync6(
|
|
5931
|
+
const path3 = join9(ensureKeysDir(), label + TOKEN_EXT);
|
|
5932
|
+
if (!existsSync9(path3)) return null;
|
|
5933
|
+
return readFileSync6(path3, "utf8").trim();
|
|
5821
5934
|
}
|
|
5822
5935
|
function writeToken(label, token) {
|
|
5823
5936
|
assertSafeLabel(label);
|
|
5824
|
-
const
|
|
5825
|
-
writeFileSync2(
|
|
5937
|
+
const path3 = join9(ensureKeysDir(), label + TOKEN_EXT);
|
|
5938
|
+
writeFileSync2(path3, token + "\n", "utf8");
|
|
5826
5939
|
if (platform2() !== "win32") {
|
|
5827
5940
|
try {
|
|
5828
|
-
chmodSync2(
|
|
5941
|
+
chmodSync2(path3, 384);
|
|
5829
5942
|
} catch {
|
|
5830
5943
|
}
|
|
5831
5944
|
}
|
|
5832
5945
|
}
|
|
5833
5946
|
function deleteToken(label) {
|
|
5834
5947
|
assertSafeLabel(label);
|
|
5835
|
-
const
|
|
5836
|
-
if (existsSync9(
|
|
5948
|
+
const path3 = join9(ensureKeysDir(), label + TOKEN_EXT);
|
|
5949
|
+
if (existsSync9(path3)) unlinkSync3(path3);
|
|
5837
5950
|
}
|
|
5838
5951
|
var MASTER_KEY_FILENAME, IDENTITY_FILENAME, EMPTY_IDENTITY, PREFERENCES_FILENAME, DEFAULT_PREFERENCES, DB_CREDENTIALS_FILENAME, CRED_LOCK_FILENAME, LOCK_STALE_MS, LOCK_TIMEOUT_MS, lockDepthInProcess, S3_CONFIG_FILENAME, ASSISTANT_CREDENTIALS_FILENAME, CLEARED_SENTINEL_PREFIX, KEYS_SUBDIR, TOKEN_EXT;
|
|
5839
5952
|
var init_user_config = __esm({
|
|
@@ -6208,13 +6321,13 @@ function uniqueDirName(displayName, existing) {
|
|
|
6208
6321
|
}
|
|
6209
6322
|
}
|
|
6210
6323
|
function readRegistry(root6) {
|
|
6211
|
-
const
|
|
6212
|
-
if (!existsSync10(
|
|
6324
|
+
const path3 = registryPath(root6);
|
|
6325
|
+
if (!existsSync10(path3)) return { ...EMPTY_REGISTRY, workspaces: [] };
|
|
6213
6326
|
let parsed;
|
|
6214
6327
|
try {
|
|
6215
|
-
parsed = JSON.parse(readFileSync8(
|
|
6328
|
+
parsed = JSON.parse(readFileSync8(path3, "utf-8"));
|
|
6216
6329
|
} catch (e6) {
|
|
6217
|
-
throw new Error(`Lattice: corrupt workspace registry at "${
|
|
6330
|
+
throw new Error(`Lattice: corrupt workspace registry at "${path3}": ${e6.message}`);
|
|
6218
6331
|
}
|
|
6219
6332
|
const reg = parsed;
|
|
6220
6333
|
return {
|
|
@@ -6224,11 +6337,11 @@ function readRegistry(root6) {
|
|
|
6224
6337
|
};
|
|
6225
6338
|
}
|
|
6226
6339
|
function writeRegistry(root6, registry) {
|
|
6227
|
-
const
|
|
6228
|
-
const tmp = `${
|
|
6340
|
+
const path3 = registryPath(root6);
|
|
6341
|
+
const tmp = `${path3}.tmp-${String(process.pid)}`;
|
|
6229
6342
|
writeFileSync3(tmp, `${JSON.stringify(registry, null, 2)}
|
|
6230
6343
|
`, "utf-8");
|
|
6231
|
-
renameSync3(tmp,
|
|
6344
|
+
renameSync3(tmp, path3);
|
|
6232
6345
|
}
|
|
6233
6346
|
function listWorkspaces(root6) {
|
|
6234
6347
|
return readRegistry(root6).workspaces;
|
|
@@ -6436,6 +6549,7 @@ function deriveCanonicalContexts(tables) {
|
|
|
6436
6549
|
childrenOf.set(rel.table, list);
|
|
6437
6550
|
}
|
|
6438
6551
|
}
|
|
6552
|
+
const byName = new Map(tables.map((t8) => [t8.name, t8.definition]));
|
|
6439
6553
|
const out = [];
|
|
6440
6554
|
for (const { name, definition } of tables) {
|
|
6441
6555
|
const files = {};
|
|
@@ -6451,11 +6565,32 @@ function deriveCanonicalContexts(tables) {
|
|
|
6451
6565
|
};
|
|
6452
6566
|
}
|
|
6453
6567
|
for (const child of childrenOf.get(name) ?? []) {
|
|
6454
|
-
|
|
6455
|
-
|
|
6456
|
-
|
|
6457
|
-
|
|
6458
|
-
|
|
6568
|
+
const childDef = byName.get(child.table);
|
|
6569
|
+
const childBt = childDef ? belongsToRelations(childDef) : [];
|
|
6570
|
+
const [rel0, rel1] = childBt;
|
|
6571
|
+
if (childDef && rel0 && rel1 && isRenderJunction(childDef, childBt)) {
|
|
6572
|
+
const localRel = rel0.foreignKey === child.foreignKey ? rel0 : rel1;
|
|
6573
|
+
const remoteRel = localRel === rel0 ? rel1 : rel0;
|
|
6574
|
+
const fileKey = remoteRel.table === name ? `${child.table.toUpperCase()}__${remoteRel.foreignKey.toUpperCase()}.md` : `${remoteRel.table.toUpperCase()}.md`;
|
|
6575
|
+
files[fileKey] = {
|
|
6576
|
+
source: {
|
|
6577
|
+
type: "manyToMany",
|
|
6578
|
+
junctionTable: child.table,
|
|
6579
|
+
localKey: localRel.foreignKey,
|
|
6580
|
+
remoteKey: remoteRel.foreignKey,
|
|
6581
|
+
remoteTable: remoteRel.table,
|
|
6582
|
+
references: remoteRel.references ?? "id"
|
|
6583
|
+
},
|
|
6584
|
+
render: renderRelated(remoteRel.table),
|
|
6585
|
+
omitIfEmpty: true
|
|
6586
|
+
};
|
|
6587
|
+
} else {
|
|
6588
|
+
files[`${child.table.toUpperCase()}.md`] = {
|
|
6589
|
+
source: { type: "hasMany", table: child.table, foreignKey: child.foreignKey },
|
|
6590
|
+
render: renderRelated(child.table),
|
|
6591
|
+
omitIfEmpty: true
|
|
6592
|
+
};
|
|
6593
|
+
}
|
|
6459
6594
|
}
|
|
6460
6595
|
out.push({
|
|
6461
6596
|
table: name,
|
|
@@ -6468,6 +6603,15 @@ function deriveCanonicalContexts(tables) {
|
|
|
6468
6603
|
}
|
|
6469
6604
|
return out;
|
|
6470
6605
|
}
|
|
6606
|
+
function isRenderJunction(def, bt) {
|
|
6607
|
+
if (bt.length !== 2) return false;
|
|
6608
|
+
const fks = new Set(bt.map((r6) => r6.foreignKey));
|
|
6609
|
+
if (fks.size !== 2) return false;
|
|
6610
|
+
const pk = Array.isArray(def.primaryKey) ? def.primaryKey : def.primaryKey != null ? [def.primaryKey] : [];
|
|
6611
|
+
if (pk.length === 2 && pk.every((c6) => fks.has(c6))) return true;
|
|
6612
|
+
const SYSTEM2 = /* @__PURE__ */ new Set(["id", "created_at", "updated_at", "deleted_at"]);
|
|
6613
|
+
return Object.keys(def.columns).every((c6) => fks.has(c6) || SYSTEM2.has(c6));
|
|
6614
|
+
}
|
|
6471
6615
|
function belongsToRelations(def) {
|
|
6472
6616
|
return Object.values(def.relations ?? {}).filter(
|
|
6473
6617
|
(r6) => r6.type === "belongsTo"
|
|
@@ -6919,6 +7063,19 @@ var init_vector_index = __esm({
|
|
|
6919
7063
|
}
|
|
6920
7064
|
});
|
|
6921
7065
|
|
|
7066
|
+
// src/search/limits.ts
|
|
7067
|
+
function clampTopK(topK) {
|
|
7068
|
+
if (!Number.isFinite(topK)) return 1;
|
|
7069
|
+
return Math.min(Math.max(1, Math.floor(topK)), SEARCH_TOPK_MAX);
|
|
7070
|
+
}
|
|
7071
|
+
var SEARCH_TOPK_MAX;
|
|
7072
|
+
var init_limits = __esm({
|
|
7073
|
+
"src/search/limits.ts"() {
|
|
7074
|
+
"use strict";
|
|
7075
|
+
SEARCH_TOPK_MAX = 1e3;
|
|
7076
|
+
}
|
|
7077
|
+
});
|
|
7078
|
+
|
|
6922
7079
|
// src/search/embeddings.ts
|
|
6923
7080
|
async function ensureEmbeddingsTable(adapter) {
|
|
6924
7081
|
let cols = [];
|
|
@@ -7065,9 +7222,10 @@ function cosineSimilarity(a6, b6) {
|
|
|
7065
7222
|
}
|
|
7066
7223
|
async function searchByEmbedding(adapter, table, queryText, config, topK, minScore, pkColumn = "id") {
|
|
7067
7224
|
const queryVector = await config.embed(queryText);
|
|
7225
|
+
const k6 = clampTopK(topK);
|
|
7068
7226
|
let ranked;
|
|
7069
7227
|
if (await vectorIndexAvailable(adapter) && await hasVectorIndex(adapter, table)) {
|
|
7070
|
-
const hits = await searchVectorIndex(adapter, table, queryVector,
|
|
7228
|
+
const hits = await searchVectorIndex(adapter, table, queryVector, k6 * 4, minScore);
|
|
7071
7229
|
ranked = hits.map((h6) => ({
|
|
7072
7230
|
pk: h6.pk,
|
|
7073
7231
|
score: h6.score,
|
|
@@ -7075,7 +7233,7 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
7075
7233
|
content: h6.content
|
|
7076
7234
|
}));
|
|
7077
7235
|
} else {
|
|
7078
|
-
ranked = await scanChunks(adapter, table, queryVector, minScore);
|
|
7236
|
+
ranked = await scanChunks(adapter, table, queryVector, minScore, config.maxScanChunks);
|
|
7079
7237
|
}
|
|
7080
7238
|
const bestByRow = /* @__PURE__ */ new Map();
|
|
7081
7239
|
for (const r6 of ranked) {
|
|
@@ -7100,11 +7258,20 @@ async function searchByEmbedding(adapter, table, queryText, config, topK, minSco
|
|
|
7100
7258
|
if (r6.content !== null) result.matchedContent = r6.content;
|
|
7101
7259
|
}
|
|
7102
7260
|
results.push(result);
|
|
7103
|
-
if (results.length >=
|
|
7261
|
+
if (results.length >= k6) break;
|
|
7104
7262
|
}
|
|
7105
7263
|
return results;
|
|
7106
7264
|
}
|
|
7107
|
-
async function scanChunks(adapter, table, queryVector, minScore) {
|
|
7265
|
+
async function scanChunks(adapter, table, queryVector, minScore, maxScanChunks) {
|
|
7266
|
+
if (maxScanChunks !== void 0) {
|
|
7267
|
+
const countRows = await allAsyncOrSync(
|
|
7268
|
+
adapter,
|
|
7269
|
+
`SELECT COUNT(*) AS n FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
7270
|
+
[table]
|
|
7271
|
+
);
|
|
7272
|
+
const n3 = Number(countRows[0]?.n ?? 0);
|
|
7273
|
+
if (n3 > maxScanChunks) throw new EmbeddingScanTooLargeError(table, n3, maxScanChunks);
|
|
7274
|
+
}
|
|
7108
7275
|
const stored = await allAsyncOrSync(
|
|
7109
7276
|
adapter,
|
|
7110
7277
|
`SELECT "row_pk", "chunk_index", "content", "embedding", "vec_dim" FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
|
|
@@ -7214,13 +7381,14 @@ async function refreshEmbeddings(adapter, table, config, pkColumn = "id", opts =
|
|
|
7214
7381
|
}
|
|
7215
7382
|
return { embedded, skipped, removed };
|
|
7216
7383
|
}
|
|
7217
|
-
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError;
|
|
7384
|
+
var EMBEDDINGS_TABLE, EmbeddingDimensionMismatchError, EmbeddingScanTooLargeError;
|
|
7218
7385
|
var init_embeddings = __esm({
|
|
7219
7386
|
"src/search/embeddings.ts"() {
|
|
7220
7387
|
"use strict";
|
|
7221
7388
|
init_adapter();
|
|
7222
7389
|
init_chunking();
|
|
7223
7390
|
init_vector_index();
|
|
7391
|
+
init_limits();
|
|
7224
7392
|
EMBEDDINGS_TABLE = "_lattice_embeddings";
|
|
7225
7393
|
EmbeddingDimensionMismatchError = class extends Error {
|
|
7226
7394
|
constructor(table, expected, found) {
|
|
@@ -7233,6 +7401,17 @@ var init_embeddings = __esm({
|
|
|
7233
7401
|
this.name = "EmbeddingDimensionMismatchError";
|
|
7234
7402
|
}
|
|
7235
7403
|
};
|
|
7404
|
+
EmbeddingScanTooLargeError = class extends Error {
|
|
7405
|
+
constructor(table, found, limit) {
|
|
7406
|
+
super(
|
|
7407
|
+
`Embedding scan on "${table}" would read ${String(found)} stored chunk vectors, over the configured maxScanChunks of ${String(limit)}. Add a native vector index (pgvector) for this table or raise maxScanChunks \u2014 Lattice will not silently truncate the scan, which would return incomplete results.`
|
|
7408
|
+
);
|
|
7409
|
+
this.table = table;
|
|
7410
|
+
this.found = found;
|
|
7411
|
+
this.limit = limit;
|
|
7412
|
+
this.name = "EmbeddingScanTooLargeError";
|
|
7413
|
+
}
|
|
7414
|
+
};
|
|
7236
7415
|
}
|
|
7237
7416
|
});
|
|
7238
7417
|
|
|
@@ -7585,7 +7764,7 @@ async function fetchLiveRows2(adapter, table, ids, pkColumn) {
|
|
|
7585
7764
|
return out;
|
|
7586
7765
|
}
|
|
7587
7766
|
async function hybridSearch(adapter, table, query, opts = {}) {
|
|
7588
|
-
const topK = opts.topK ?? 10;
|
|
7767
|
+
const topK = clampTopK(opts.topK ?? 10);
|
|
7589
7768
|
const rrfK = opts.rrfK ?? 60;
|
|
7590
7769
|
const pool = opts.poolSize ?? Math.max(topK * 4, 20);
|
|
7591
7770
|
const pkColumn = opts.pkColumn ?? "id";
|
|
@@ -7686,6 +7865,7 @@ var init_hybrid = __esm({
|
|
|
7686
7865
|
init_fts();
|
|
7687
7866
|
init_ranking();
|
|
7688
7867
|
init_rerank();
|
|
7868
|
+
init_limits();
|
|
7689
7869
|
}
|
|
7690
7870
|
});
|
|
7691
7871
|
|
|
@@ -7942,18 +8122,18 @@ function computedColumnOrder(table, computed) {
|
|
|
7942
8122
|
const names = new Set(Object.keys(computed));
|
|
7943
8123
|
const order = [];
|
|
7944
8124
|
const state2 = /* @__PURE__ */ new Map();
|
|
7945
|
-
const visit = (name,
|
|
8125
|
+
const visit = (name, path3) => {
|
|
7946
8126
|
const st = state2.get(name);
|
|
7947
8127
|
if (st === "done") return;
|
|
7948
8128
|
if (st === "visiting") {
|
|
7949
|
-
const start =
|
|
7950
|
-
throw new ComputedColumnCycleError(table, [...
|
|
8129
|
+
const start = path3.indexOf(name);
|
|
8130
|
+
throw new ComputedColumnCycleError(table, [...path3.slice(start), name]);
|
|
7951
8131
|
}
|
|
7952
8132
|
state2.set(name, "visiting");
|
|
7953
8133
|
const spec = computed[name];
|
|
7954
8134
|
if (spec) {
|
|
7955
8135
|
for (const dep of spec.deps) {
|
|
7956
|
-
if (names.has(dep)) visit(dep, [...
|
|
8136
|
+
if (names.has(dep)) visit(dep, [...path3, name]);
|
|
7957
8137
|
}
|
|
7958
8138
|
}
|
|
7959
8139
|
state2.set(name, "done");
|
|
@@ -8413,6 +8593,26 @@ RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
|
8413
8593
|
);
|
|
8414
8594
|
$fn$;
|
|
8415
8595
|
|
|
8596
|
+
-- Delete-event visibility, decided from the PRE-DELETE snapshot the delete trigger
|
|
8597
|
+
-- captures (the live row + its ownership record are gone after a delete, so
|
|
8598
|
+
-- lattice_row_visible can't be used). Keyed on session_user, SECURITY DEFINER \u2014
|
|
8599
|
+
-- the same per-recipient gate. MUST MIRROR lattice_row_visible's rule: the row is
|
|
8600
|
+
-- visible iff this member owned it, OR it was 'everyone', OR it was 'custom' and
|
|
8601
|
+
-- this member was a grantee. A NULL owner snapshot (a legacy delete emitted before
|
|
8602
|
+
-- the snapshot columns, or a row with no ownership record) yields false \u2014 fail
|
|
8603
|
+
-- closed, never forward. (tests/integration assert this agrees with
|
|
8604
|
+
-- lattice_row_visible for all three visibility states \u2014 the no-drift guard.)
|
|
8605
|
+
CREATE OR REPLACE FUNCTION lattice_delete_visible(
|
|
8606
|
+
p_owner_role text, p_visibility text, p_grantees text[]
|
|
8607
|
+
)
|
|
8608
|
+
RETURNS boolean LANGUAGE sql STABLE SECURITY DEFINER AS $fn$
|
|
8609
|
+
SELECT p_owner_role IS NOT NULL AND (
|
|
8610
|
+
p_owner_role = session_user
|
|
8611
|
+
OR p_visibility = 'everyone'
|
|
8612
|
+
OR (p_visibility = 'custom' AND session_user = ANY(COALESCE(p_grantees, ARRAY[]::text[])))
|
|
8613
|
+
);
|
|
8614
|
+
$fn$;
|
|
8615
|
+
|
|
8416
8616
|
-- Shared owner gate: raises unless the connected member owns (p_table, p_pk).
|
|
8417
8617
|
-- p_action is spliced into the message so every caller keeps its exact wording.
|
|
8418
8618
|
-- SECURITY DEFINER + session_user (never current_user), the cloud identity invariant.
|
|
@@ -8587,6 +8787,14 @@ CREATE TABLE IF NOT EXISTS "__lattice_changes" (
|
|
|
8587
8787
|
"created_at" timestamptz NOT NULL DEFAULT now()
|
|
8588
8788
|
);
|
|
8589
8789
|
|
|
8790
|
+
-- Pre-delete visibility snapshot columns (added to existing clouds via ADD COLUMN
|
|
8791
|
+
-- IF NOT EXISTS). A delete event carries the row's visibility AT DELETE TIME so the
|
|
8792
|
+
-- live fan-out can gate it per recipient even though the ownership record is gone.
|
|
8793
|
+
-- NULL on upserts.
|
|
8794
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_owner_role" text;
|
|
8795
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_visibility" text;
|
|
8796
|
+
ALTER TABLE "__lattice_changes" ADD COLUMN IF NOT EXISTS "del_grantees" text[];
|
|
8797
|
+
|
|
8590
8798
|
CREATE OR REPLACE FUNCTION lattice_notify_change() RETURNS trigger
|
|
8591
8799
|
LANGUAGE plpgsql AS $fn$
|
|
8592
8800
|
BEGIN
|
|
@@ -8596,7 +8804,10 @@ BEGIN
|
|
|
8596
8804
|
'pk', NEW."pk",
|
|
8597
8805
|
'op', NEW."op",
|
|
8598
8806
|
'owner_role', NEW."owner_role",
|
|
8599
|
-
'created_at', NEW."created_at"
|
|
8807
|
+
'created_at', NEW."created_at",
|
|
8808
|
+
'del_owner_role', NEW."del_owner_role",
|
|
8809
|
+
'del_visibility', NEW."del_visibility",
|
|
8810
|
+
'del_grantees', NEW."del_grantees"
|
|
8600
8811
|
)::text);
|
|
8601
8812
|
RETURN NEW;
|
|
8602
8813
|
END $fn$;
|
|
@@ -8802,10 +9013,22 @@ BEGIN
|
|
|
8802
9013
|
VALUES (${lit}, ${pkNew}, 'upsert', session_user);
|
|
8803
9014
|
RETURN NEW;
|
|
8804
9015
|
ELSIF TG_OP = 'DELETE' THEN
|
|
9016
|
+
-- Snapshot the row's visibility BEFORE the cascade removes its ownership +
|
|
9017
|
+
-- grant records, so the realtime fan-out can gate the delete event per
|
|
9018
|
+
-- recipient (the live predicate can't \u2014 these records are gone post-delete).
|
|
9019
|
+
-- The grantee list is captured here because the grant rows are deleted in the
|
|
9020
|
+
-- same statement below; after that the 'custom' audience is unrecoverable.
|
|
9021
|
+
INSERT INTO "__lattice_changes"
|
|
9022
|
+
("table_name","pk","op","owner_role","del_owner_role","del_visibility","del_grantees")
|
|
9023
|
+
VALUES (${lit}, ${pkOld}, 'delete', session_user,
|
|
9024
|
+
(SELECT o."owner_role" FROM "__lattice_owners" o
|
|
9025
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
9026
|
+
(SELECT o."visibility" FROM "__lattice_owners" o
|
|
9027
|
+
WHERE o."table_name" = ${lit} AND o."pk" = ${pkOld}),
|
|
9028
|
+
COALESCE((SELECT array_agg(g."grantee_role") FROM "__lattice_row_grants" g
|
|
9029
|
+
WHERE g."table_name" = ${lit} AND g."pk" = ${pkOld}), ARRAY[]::text[]));
|
|
8805
9030
|
DELETE FROM "__lattice_owners" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8806
9031
|
DELETE FROM "__lattice_row_grants" WHERE "table_name" = ${lit} AND "pk" = ${pkOld};
|
|
8807
|
-
INSERT INTO "__lattice_changes" ("table_name","pk","op","owner_role")
|
|
8808
|
-
VALUES (${lit}, ${pkOld}, 'delete', session_user);
|
|
8809
9032
|
RETURN OLD;
|
|
8810
9033
|
END IF;
|
|
8811
9034
|
RETURN NEW;
|
|
@@ -13410,7 +13633,7 @@ var init_sleep = __esm({
|
|
|
13410
13633
|
"node_modules/@smithy/core/dist-es/submodules/client/util-waiter/utils/sleep.js"() {
|
|
13411
13634
|
"use strict";
|
|
13412
13635
|
sleep = (seconds) => {
|
|
13413
|
-
return new Promise((
|
|
13636
|
+
return new Promise((resolve17) => setTimeout(resolve17, seconds * 1e3));
|
|
13414
13637
|
};
|
|
13415
13638
|
}
|
|
13416
13639
|
});
|
|
@@ -13579,8 +13802,8 @@ var init_createWaiter = __esm({
|
|
|
13579
13802
|
init_waiter2();
|
|
13580
13803
|
abortTimeout = (abortSignal) => {
|
|
13581
13804
|
let onAbort;
|
|
13582
|
-
const promise = new Promise((
|
|
13583
|
-
onAbort = () =>
|
|
13805
|
+
const promise = new Promise((resolve17) => {
|
|
13806
|
+
onAbort = () => resolve17({ state: WaiterState.ABORTED });
|
|
13584
13807
|
if (typeof abortSignal.addEventListener === "function") {
|
|
13585
13808
|
abortSignal.addEventListener("abort", onAbort);
|
|
13586
13809
|
} else {
|
|
@@ -16000,14 +16223,14 @@ var init_readFile = __esm({
|
|
|
16000
16223
|
"use strict";
|
|
16001
16224
|
filePromises = {};
|
|
16002
16225
|
fileIntercept = {};
|
|
16003
|
-
readFile2 = (
|
|
16004
|
-
if (fileIntercept[
|
|
16005
|
-
return fileIntercept[
|
|
16226
|
+
readFile2 = (path3, options) => {
|
|
16227
|
+
if (fileIntercept[path3] !== void 0) {
|
|
16228
|
+
return fileIntercept[path3];
|
|
16006
16229
|
}
|
|
16007
|
-
if (!filePromises[
|
|
16008
|
-
filePromises[
|
|
16230
|
+
if (!filePromises[path3] || options?.ignoreCache) {
|
|
16231
|
+
filePromises[path3] = fsReadFile(path3, "utf8");
|
|
16009
16232
|
}
|
|
16010
|
-
return filePromises[
|
|
16233
|
+
return filePromises[path3];
|
|
16011
16234
|
};
|
|
16012
16235
|
}
|
|
16013
16236
|
});
|
|
@@ -16125,8 +16348,8 @@ var init_externalDataInterceptor = __esm({
|
|
|
16125
16348
|
getFileRecord() {
|
|
16126
16349
|
return fileIntercept;
|
|
16127
16350
|
},
|
|
16128
|
-
interceptFile(
|
|
16129
|
-
fileIntercept[
|
|
16351
|
+
interceptFile(path3, contents) {
|
|
16352
|
+
fileIntercept[path3] = Promise.resolve(contents);
|
|
16130
16353
|
},
|
|
16131
16354
|
getTokenRecord() {
|
|
16132
16355
|
return tokenIntercept;
|
|
@@ -16460,13 +16683,13 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
16460
16683
|
}
|
|
16461
16684
|
return { hostname: "169.254.169.254", path: "/" };
|
|
16462
16685
|
};
|
|
16463
|
-
imdsHttpGet = async ({ hostname, path:
|
|
16686
|
+
imdsHttpGet = async ({ hostname, path: path3 }) => {
|
|
16464
16687
|
const { request } = await import("http");
|
|
16465
|
-
return new Promise((
|
|
16688
|
+
return new Promise((resolve17, reject) => {
|
|
16466
16689
|
const req = request({
|
|
16467
16690
|
method: "GET",
|
|
16468
16691
|
hostname: hostname.replace(/^\[(.+)]$/, "$1"),
|
|
16469
|
-
path:
|
|
16692
|
+
path: path3,
|
|
16470
16693
|
timeout: 1e3,
|
|
16471
16694
|
signal: AbortSignal.timeout(1e3)
|
|
16472
16695
|
});
|
|
@@ -16488,7 +16711,7 @@ var init_resolveDefaultsModeConfig = __esm({
|
|
|
16488
16711
|
const chunks = [];
|
|
16489
16712
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
16490
16713
|
res.on("end", () => {
|
|
16491
|
-
|
|
16714
|
+
resolve17(Buffer.concat(chunks));
|
|
16492
16715
|
req.destroy();
|
|
16493
16716
|
});
|
|
16494
16717
|
});
|
|
@@ -16667,8 +16890,8 @@ var init_createConfigValueProvider = __esm({
|
|
|
16667
16890
|
return endpoint.url.href;
|
|
16668
16891
|
}
|
|
16669
16892
|
if ("hostname" in endpoint) {
|
|
16670
|
-
const { protocol, hostname, port, path:
|
|
16671
|
-
return `${protocol}//${hostname}${port ? ":" + port : ""}${
|
|
16893
|
+
const { protocol, hostname, port, path: path3 } = endpoint;
|
|
16894
|
+
return `${protocol}//${hostname}${port ? ":" + port : ""}${path3}`;
|
|
16672
16895
|
}
|
|
16673
16896
|
}
|
|
16674
16897
|
return endpoint;
|
|
@@ -17105,18 +17328,18 @@ var init_getAttrPathList = __esm({
|
|
|
17105
17328
|
"node_modules/@smithy/core/dist-es/submodules/endpoints/util-endpoints/lib/getAttrPathList.js"() {
|
|
17106
17329
|
"use strict";
|
|
17107
17330
|
init_types2();
|
|
17108
|
-
getAttrPathList = (
|
|
17109
|
-
const parts =
|
|
17331
|
+
getAttrPathList = (path3) => {
|
|
17332
|
+
const parts = path3.split(".");
|
|
17110
17333
|
const pathList = [];
|
|
17111
17334
|
for (const part of parts) {
|
|
17112
17335
|
const squareBracketIndex = part.indexOf("[");
|
|
17113
17336
|
if (squareBracketIndex !== -1) {
|
|
17114
17337
|
if (part.indexOf("]") !== part.length - 1) {
|
|
17115
|
-
throw new EndpointError(`Path: '${
|
|
17338
|
+
throw new EndpointError(`Path: '${path3}' does not end with ']'`);
|
|
17116
17339
|
}
|
|
17117
17340
|
const arrayIndex = part.slice(squareBracketIndex + 1, -1);
|
|
17118
17341
|
if (Number.isNaN(parseInt(arrayIndex))) {
|
|
17119
|
-
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${
|
|
17342
|
+
throw new EndpointError(`Invalid array index: '${arrayIndex}' in path: '${path3}'`);
|
|
17120
17343
|
}
|
|
17121
17344
|
if (squareBracketIndex !== 0) {
|
|
17122
17345
|
pathList.push(part.slice(0, squareBracketIndex));
|
|
@@ -17138,9 +17361,9 @@ var init_getAttr = __esm({
|
|
|
17138
17361
|
"use strict";
|
|
17139
17362
|
init_types2();
|
|
17140
17363
|
init_getAttrPathList();
|
|
17141
|
-
getAttr = (value,
|
|
17364
|
+
getAttr = (value, path3) => getAttrPathList(path3).reduce((acc, index) => {
|
|
17142
17365
|
if (typeof acc !== "object") {
|
|
17143
|
-
throw new EndpointError(`Index '${index}' in '${
|
|
17366
|
+
throw new EndpointError(`Index '${index}' in '${path3}' not found in '${JSON.stringify(value)}'`);
|
|
17144
17367
|
} else if (Array.isArray(acc)) {
|
|
17145
17368
|
const i6 = parseInt(index);
|
|
17146
17369
|
return acc[i6 < 0 ? acc.length + i6 : i6];
|
|
@@ -17206,8 +17429,8 @@ var init_parseURL = __esm({
|
|
|
17206
17429
|
return value;
|
|
17207
17430
|
}
|
|
17208
17431
|
if (typeof value === "object" && "hostname" in value) {
|
|
17209
|
-
const { hostname: hostname2, port, protocol: protocol2 = "", path:
|
|
17210
|
-
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${
|
|
17432
|
+
const { hostname: hostname2, port, protocol: protocol2 = "", path: path3 = "", query = {} } = value;
|
|
17433
|
+
const url = new URL(`${protocol2}//${hostname2}${port ? `:${port}` : ""}${path3}`);
|
|
17211
17434
|
url.search = Object.entries(query).map(([k6, v2]) => `${k6}=${v2}`).join("&");
|
|
17212
17435
|
return url;
|
|
17213
17436
|
}
|
|
@@ -18248,7 +18471,7 @@ async function collectStream(stream) {
|
|
|
18248
18471
|
return collected;
|
|
18249
18472
|
}
|
|
18250
18473
|
function readToBase64(blob) {
|
|
18251
|
-
return new Promise((
|
|
18474
|
+
return new Promise((resolve17, reject) => {
|
|
18252
18475
|
const reader = new FileReader();
|
|
18253
18476
|
reader.onloadend = () => {
|
|
18254
18477
|
if (reader.readyState !== 2) {
|
|
@@ -18257,7 +18480,7 @@ function readToBase64(blob) {
|
|
|
18257
18480
|
const result = reader.result ?? "";
|
|
18258
18481
|
const commaIndex = result.indexOf(",");
|
|
18259
18482
|
const dataOffset = commaIndex > -1 ? commaIndex + 1 : result.length;
|
|
18260
|
-
|
|
18483
|
+
resolve17(result.substring(dataOffset));
|
|
18261
18484
|
};
|
|
18262
18485
|
reader.onabort = () => reject(new Error("Read aborted"));
|
|
18263
18486
|
reader.onerror = () => reject(reader.error);
|
|
@@ -18385,7 +18608,7 @@ var init_stream_collector = __esm({
|
|
|
18385
18608
|
if (isReadableStreamInstance(stream)) {
|
|
18386
18609
|
return collectReadableStream(stream);
|
|
18387
18610
|
}
|
|
18388
|
-
return new Promise((
|
|
18611
|
+
return new Promise((resolve17, reject) => {
|
|
18389
18612
|
const collector = new Collector();
|
|
18390
18613
|
stream.pipe(collector);
|
|
18391
18614
|
stream.on("error", (err) => {
|
|
@@ -18395,7 +18618,7 @@ var init_stream_collector = __esm({
|
|
|
18395
18618
|
collector.on("error", reject);
|
|
18396
18619
|
collector.on("finish", function() {
|
|
18397
18620
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
18398
|
-
|
|
18621
|
+
resolve17(bytes);
|
|
18399
18622
|
});
|
|
18400
18623
|
});
|
|
18401
18624
|
};
|
|
@@ -18542,11 +18765,11 @@ var init_SerdeContext = __esm({
|
|
|
18542
18765
|
// node_modules/tslib/tslib.es6.mjs
|
|
18543
18766
|
function __awaiter(thisArg, _arguments, P2, generator) {
|
|
18544
18767
|
function adopt(value) {
|
|
18545
|
-
return value instanceof P2 ? value : new P2(function(
|
|
18546
|
-
|
|
18768
|
+
return value instanceof P2 ? value : new P2(function(resolve17) {
|
|
18769
|
+
resolve17(value);
|
|
18547
18770
|
});
|
|
18548
18771
|
}
|
|
18549
|
-
return new (P2 || (P2 = Promise))(function(
|
|
18772
|
+
return new (P2 || (P2 = Promise))(function(resolve17, reject) {
|
|
18550
18773
|
function fulfilled(value) {
|
|
18551
18774
|
try {
|
|
18552
18775
|
step(generator.next(value));
|
|
@@ -18562,7 +18785,7 @@ function __awaiter(thisArg, _arguments, P2, generator) {
|
|
|
18562
18785
|
}
|
|
18563
18786
|
}
|
|
18564
18787
|
function step(result) {
|
|
18565
|
-
result.done ?
|
|
18788
|
+
result.done ? resolve17(result.value) : adopt(result.value).then(fulfilled, rejected);
|
|
18566
18789
|
}
|
|
18567
18790
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
18568
18791
|
});
|
|
@@ -19759,7 +19982,7 @@ async function* readableToIterable(readStream) {
|
|
|
19759
19982
|
streamEnded = true;
|
|
19760
19983
|
});
|
|
19761
19984
|
while (!generationEnded) {
|
|
19762
|
-
const value = await new Promise((
|
|
19985
|
+
const value = await new Promise((resolve17) => setTimeout(() => resolve17(records.shift()), 0));
|
|
19763
19986
|
if (value) {
|
|
19764
19987
|
yield value;
|
|
19765
19988
|
}
|
|
@@ -20318,11 +20541,11 @@ var init_HttpBindingProtocol = __esm({
|
|
|
20318
20541
|
const opTraits = translateTraits(operationSchema.traits);
|
|
20319
20542
|
if (opTraits.http) {
|
|
20320
20543
|
request.method = opTraits.http[0];
|
|
20321
|
-
const [
|
|
20544
|
+
const [path3, search] = opTraits.http[1].split("?");
|
|
20322
20545
|
if (request.path == "/") {
|
|
20323
|
-
request.path =
|
|
20546
|
+
request.path = path3;
|
|
20324
20547
|
} else {
|
|
20325
|
-
request.path +=
|
|
20548
|
+
request.path += path3;
|
|
20326
20549
|
}
|
|
20327
20550
|
const traitSearchParams = new URLSearchParams(search ?? "");
|
|
20328
20551
|
for (const [key, value] of traitSearchParams) {
|
|
@@ -21304,7 +21527,7 @@ var init_retryMiddleware = __esm({
|
|
|
21304
21527
|
init_constants5();
|
|
21305
21528
|
init_parseRetryAfterHeader();
|
|
21306
21529
|
init_util2();
|
|
21307
|
-
cooldown = (ms) => new Promise((
|
|
21530
|
+
cooldown = (ms) => new Promise((resolve17) => setTimeout(resolve17, ms));
|
|
21308
21531
|
isRetryStrategyV2 = (retryStrategy) => typeof retryStrategy.acquireInitialRetryToken !== "undefined" && typeof retryStrategy.refreshRetryTokenForRetry !== "undefined" && typeof retryStrategy.recordSuccess !== "undefined";
|
|
21309
21532
|
getRetryErrorInfo = (error, logger2) => {
|
|
21310
21533
|
const errorInfo = {
|
|
@@ -21403,7 +21626,7 @@ var init_DefaultRateLimiter = __esm({
|
|
|
21403
21626
|
this.refillTokenBucket();
|
|
21404
21627
|
while (amount > this.availableTokens) {
|
|
21405
21628
|
const delay = (amount - this.availableTokens) / this.fillRate * 1e3;
|
|
21406
|
-
await new Promise((
|
|
21629
|
+
await new Promise((resolve17) => _DefaultRateLimiter.setTimeoutFn(resolve17, delay));
|
|
21407
21630
|
this.refillTokenBucket();
|
|
21408
21631
|
}
|
|
21409
21632
|
this.availableTokens = this.availableTokens - amount;
|
|
@@ -22301,9 +22524,9 @@ var init_createPaginator = __esm({
|
|
|
22301
22524
|
command = withCommand(command) ?? command;
|
|
22302
22525
|
return await client.send(command, ...args);
|
|
22303
22526
|
};
|
|
22304
|
-
get = (fromObject,
|
|
22527
|
+
get = (fromObject, path3) => {
|
|
22305
22528
|
let cursor = fromObject;
|
|
22306
|
-
const pathComponents =
|
|
22529
|
+
const pathComponents = path3.split(".");
|
|
22307
22530
|
for (const step of pathComponents) {
|
|
22308
22531
|
if (!cursor || typeof cursor !== "object") {
|
|
22309
22532
|
return void 0;
|
|
@@ -24795,10 +25018,10 @@ ${longDate}
|
|
|
24795
25018
|
${credentialScope}
|
|
24796
25019
|
${toHex(hashedRequest)}`;
|
|
24797
25020
|
}
|
|
24798
|
-
getCanonicalPath({ path:
|
|
25021
|
+
getCanonicalPath({ path: path3 }) {
|
|
24799
25022
|
if (this.uriEscapePath) {
|
|
24800
25023
|
const normalizedPathSegments = [];
|
|
24801
|
-
for (const pathSegment of
|
|
25024
|
+
for (const pathSegment of path3.split("/")) {
|
|
24802
25025
|
if (pathSegment?.length === 0)
|
|
24803
25026
|
continue;
|
|
24804
25027
|
if (pathSegment === ".")
|
|
@@ -24809,11 +25032,11 @@ ${toHex(hashedRequest)}`;
|
|
|
24809
25032
|
normalizedPathSegments.push(pathSegment);
|
|
24810
25033
|
}
|
|
24811
25034
|
}
|
|
24812
|
-
const normalizedPath = `${
|
|
25035
|
+
const normalizedPath = `${path3?.startsWith("/") ? "/" : ""}${normalizedPathSegments.join("/")}${normalizedPathSegments.length > 0 && path3?.endsWith("/") ? "/" : ""}`;
|
|
24813
25036
|
const doubleEncoded = escapeUri(normalizedPath);
|
|
24814
25037
|
return doubleEncoded.replace(/%2F/g, "/");
|
|
24815
25038
|
}
|
|
24816
|
-
return
|
|
25039
|
+
return path3;
|
|
24817
25040
|
}
|
|
24818
25041
|
validateResolvedCredentials(credentials) {
|
|
24819
25042
|
if (typeof credentials !== "object" || typeof credentials.accessKeyId !== "string" || typeof credentials.secretAccessKey !== "string") {
|
|
@@ -25073,8 +25296,8 @@ var init_SignatureV4 = __esm({
|
|
|
25073
25296
|
priorSignature: signableMessage.priorSignature,
|
|
25074
25297
|
eventStreamCredentials
|
|
25075
25298
|
});
|
|
25076
|
-
return promise.then((
|
|
25077
|
-
return { message: signableMessage.message, signature };
|
|
25299
|
+
return promise.then((signature2) => {
|
|
25300
|
+
return { message: signableMessage.message, signature: signature2 };
|
|
25078
25301
|
});
|
|
25079
25302
|
}
|
|
25080
25303
|
async signString(stringToSign, { signingDate = /* @__PURE__ */ new Date(), signingRegion, signingService, eventStreamCredentials } = {}) {
|
|
@@ -25102,8 +25325,8 @@ var init_SignatureV4 = __esm({
|
|
|
25102
25325
|
request.headers[SHA256_HEADER] = payloadHash;
|
|
25103
25326
|
}
|
|
25104
25327
|
const canonicalHeaders = getCanonicalHeaders(request, unsignableHeaders, signableHeaders);
|
|
25105
|
-
const
|
|
25106
|
-
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${
|
|
25328
|
+
const signature2 = await this.getSignature(longDate, scope, this.getSigningKey(credentials, region, shortDate, signingService), this.createCanonicalRequest(request, canonicalHeaders, payloadHash));
|
|
25329
|
+
request.headers[AUTH_HEADER] = `${ALGORITHM_IDENTIFIER} Credential=${credentials.accessKeyId}/${scope}, SignedHeaders=${this.getCanonicalHeaderList(canonicalHeaders)}, Signature=${signature2}`;
|
|
25107
25330
|
return request;
|
|
25108
25331
|
}
|
|
25109
25332
|
async getSignature(longDate, credentialScope, keyPromise, canonicalRequest) {
|
|
@@ -29730,16 +29953,16 @@ var init_Matcher = __esm({
|
|
|
29730
29953
|
* @returns {string|undefined}
|
|
29731
29954
|
*/
|
|
29732
29955
|
getCurrentTag() {
|
|
29733
|
-
const
|
|
29734
|
-
return
|
|
29956
|
+
const path3 = this._matcher.path;
|
|
29957
|
+
return path3.length > 0 ? path3[path3.length - 1].tag : void 0;
|
|
29735
29958
|
}
|
|
29736
29959
|
/**
|
|
29737
29960
|
* Get current namespace.
|
|
29738
29961
|
* @returns {string|undefined}
|
|
29739
29962
|
*/
|
|
29740
29963
|
getCurrentNamespace() {
|
|
29741
|
-
const
|
|
29742
|
-
return
|
|
29964
|
+
const path3 = this._matcher.path;
|
|
29965
|
+
return path3.length > 0 ? path3[path3.length - 1].namespace : void 0;
|
|
29743
29966
|
}
|
|
29744
29967
|
/**
|
|
29745
29968
|
* Get current node's attribute value.
|
|
@@ -29747,9 +29970,9 @@ var init_Matcher = __esm({
|
|
|
29747
29970
|
* @returns {*}
|
|
29748
29971
|
*/
|
|
29749
29972
|
getAttrValue(attrName) {
|
|
29750
|
-
const
|
|
29751
|
-
if (
|
|
29752
|
-
return
|
|
29973
|
+
const path3 = this._matcher.path;
|
|
29974
|
+
if (path3.length === 0) return void 0;
|
|
29975
|
+
return path3[path3.length - 1].values?.[attrName];
|
|
29753
29976
|
}
|
|
29754
29977
|
/**
|
|
29755
29978
|
* Check if current node has an attribute.
|
|
@@ -29757,9 +29980,9 @@ var init_Matcher = __esm({
|
|
|
29757
29980
|
* @returns {boolean}
|
|
29758
29981
|
*/
|
|
29759
29982
|
hasAttr(attrName) {
|
|
29760
|
-
const
|
|
29761
|
-
if (
|
|
29762
|
-
const current =
|
|
29983
|
+
const path3 = this._matcher.path;
|
|
29984
|
+
if (path3.length === 0) return false;
|
|
29985
|
+
const current = path3[path3.length - 1];
|
|
29763
29986
|
return current.values !== void 0 && attrName in current.values;
|
|
29764
29987
|
}
|
|
29765
29988
|
/**
|
|
@@ -29767,18 +29990,18 @@ var init_Matcher = __esm({
|
|
|
29767
29990
|
* @returns {number}
|
|
29768
29991
|
*/
|
|
29769
29992
|
getPosition() {
|
|
29770
|
-
const
|
|
29771
|
-
if (
|
|
29772
|
-
return
|
|
29993
|
+
const path3 = this._matcher.path;
|
|
29994
|
+
if (path3.length === 0) return -1;
|
|
29995
|
+
return path3[path3.length - 1].position ?? 0;
|
|
29773
29996
|
}
|
|
29774
29997
|
/**
|
|
29775
29998
|
* Get current node's repeat counter (occurrence count of this tag name).
|
|
29776
29999
|
* @returns {number}
|
|
29777
30000
|
*/
|
|
29778
30001
|
getCounter() {
|
|
29779
|
-
const
|
|
29780
|
-
if (
|
|
29781
|
-
return
|
|
30002
|
+
const path3 = this._matcher.path;
|
|
30003
|
+
if (path3.length === 0) return -1;
|
|
30004
|
+
return path3[path3.length - 1].counter ?? 0;
|
|
29782
30005
|
}
|
|
29783
30006
|
/**
|
|
29784
30007
|
* Get current node's sibling index (alias for getPosition).
|
|
@@ -40691,7 +40914,7 @@ var init_node_http = __esm({
|
|
|
40691
40914
|
|
|
40692
40915
|
// node_modules/@smithy/credential-provider-imds/dist-es/remoteProvider/httpRequest.js
|
|
40693
40916
|
function httpRequest(options) {
|
|
40694
|
-
return new Promise((
|
|
40917
|
+
return new Promise((resolve17, reject) => {
|
|
40695
40918
|
const req = node_http.request({
|
|
40696
40919
|
method: "GET",
|
|
40697
40920
|
...options,
|
|
@@ -40716,7 +40939,7 @@ function httpRequest(options) {
|
|
|
40716
40939
|
chunks.push(chunk);
|
|
40717
40940
|
});
|
|
40718
40941
|
res.on("end", () => {
|
|
40719
|
-
|
|
40942
|
+
resolve17(Buffer.concat(chunks));
|
|
40720
40943
|
req.destroy();
|
|
40721
40944
|
});
|
|
40722
40945
|
});
|
|
@@ -41361,21 +41584,21 @@ async function writeRequestBody(httpRequest2, request, maxContinueTimeoutMs = MI
|
|
|
41361
41584
|
let sendBody = true;
|
|
41362
41585
|
if (!externalAgent && expect === "100-continue") {
|
|
41363
41586
|
sendBody = await Promise.race([
|
|
41364
|
-
new Promise((
|
|
41365
|
-
timeoutId = Number(timing.setTimeout(() =>
|
|
41587
|
+
new Promise((resolve17) => {
|
|
41588
|
+
timeoutId = Number(timing.setTimeout(() => resolve17(true), Math.max(MIN_WAIT_TIME, maxContinueTimeoutMs)));
|
|
41366
41589
|
}),
|
|
41367
|
-
new Promise((
|
|
41590
|
+
new Promise((resolve17) => {
|
|
41368
41591
|
httpRequest2.on("continue", () => {
|
|
41369
41592
|
timing.clearTimeout(timeoutId);
|
|
41370
|
-
|
|
41593
|
+
resolve17(true);
|
|
41371
41594
|
});
|
|
41372
41595
|
httpRequest2.on("response", () => {
|
|
41373
41596
|
timing.clearTimeout(timeoutId);
|
|
41374
|
-
|
|
41597
|
+
resolve17(false);
|
|
41375
41598
|
});
|
|
41376
41599
|
httpRequest2.on("error", () => {
|
|
41377
41600
|
timing.clearTimeout(timeoutId);
|
|
41378
|
-
|
|
41601
|
+
resolve17(false);
|
|
41379
41602
|
});
|
|
41380
41603
|
})
|
|
41381
41604
|
]);
|
|
@@ -41473,13 +41696,13 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41473
41696
|
return socketWarningTimestamp;
|
|
41474
41697
|
}
|
|
41475
41698
|
constructor(options) {
|
|
41476
|
-
this.configProvider = new Promise((
|
|
41699
|
+
this.configProvider = new Promise((resolve17, reject) => {
|
|
41477
41700
|
if (typeof options === "function") {
|
|
41478
41701
|
options().then((_options) => {
|
|
41479
|
-
|
|
41702
|
+
resolve17(this.resolveDefaultConfig(_options));
|
|
41480
41703
|
}).catch(reject);
|
|
41481
41704
|
} else {
|
|
41482
|
-
|
|
41705
|
+
resolve17(this.resolveDefaultConfig(options));
|
|
41483
41706
|
}
|
|
41484
41707
|
});
|
|
41485
41708
|
}
|
|
@@ -41510,7 +41733,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41510
41733
|
timing.clearTimeout(socketTimeoutId);
|
|
41511
41734
|
timing.clearTimeout(keepAliveTimeoutId);
|
|
41512
41735
|
};
|
|
41513
|
-
const
|
|
41736
|
+
const resolve17 = async (arg) => {
|
|
41514
41737
|
await writeRequestBodyPromise;
|
|
41515
41738
|
clearTimeouts();
|
|
41516
41739
|
_resolve(arg);
|
|
@@ -41544,12 +41767,12 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41544
41767
|
const password = request.password ?? "";
|
|
41545
41768
|
auth = `${username}:${password}`;
|
|
41546
41769
|
}
|
|
41547
|
-
let
|
|
41770
|
+
let path3 = request.path;
|
|
41548
41771
|
if (queryString) {
|
|
41549
|
-
|
|
41772
|
+
path3 += `?${queryString}`;
|
|
41550
41773
|
}
|
|
41551
41774
|
if (request.fragment) {
|
|
41552
|
-
|
|
41775
|
+
path3 += `#${request.fragment}`;
|
|
41553
41776
|
}
|
|
41554
41777
|
let hostname = request.hostname ?? "";
|
|
41555
41778
|
if (hostname[0] === "[" && hostname.endsWith("]")) {
|
|
@@ -41561,7 +41784,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41561
41784
|
headers: request.headers,
|
|
41562
41785
|
host: hostname,
|
|
41563
41786
|
method: request.method,
|
|
41564
|
-
path:
|
|
41787
|
+
path: path3,
|
|
41565
41788
|
port: request.port,
|
|
41566
41789
|
agent,
|
|
41567
41790
|
auth
|
|
@@ -41574,7 +41797,7 @@ or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler conf
|
|
|
41574
41797
|
headers: getTransformedHeaders(res.headers),
|
|
41575
41798
|
body: res
|
|
41576
41799
|
});
|
|
41577
|
-
|
|
41800
|
+
resolve17({ response: httpResponse });
|
|
41578
41801
|
});
|
|
41579
41802
|
req.on("error", (err) => {
|
|
41580
41803
|
if (NODEJS_TIMEOUT_ERROR_CODES2.includes(err.code)) {
|
|
@@ -41726,7 +41949,7 @@ var init_stream_collector2 = __esm({
|
|
|
41726
41949
|
if (isReadableStreamInstance2(stream)) {
|
|
41727
41950
|
return collectReadableStream2(stream);
|
|
41728
41951
|
}
|
|
41729
|
-
return new Promise((
|
|
41952
|
+
return new Promise((resolve17, reject) => {
|
|
41730
41953
|
const collector = new Collector2();
|
|
41731
41954
|
stream.pipe(collector);
|
|
41732
41955
|
stream.on("error", (err) => {
|
|
@@ -41736,7 +41959,7 @@ var init_stream_collector2 = __esm({
|
|
|
41736
41959
|
collector.on("error", reject);
|
|
41737
41960
|
collector.on("finish", function() {
|
|
41738
41961
|
const bytes = new Uint8Array(Buffer.concat(this.bufferedBytes));
|
|
41739
|
-
|
|
41962
|
+
resolve17(bytes);
|
|
41740
41963
|
});
|
|
41741
41964
|
});
|
|
41742
41965
|
};
|
|
@@ -41858,7 +42081,7 @@ var init_retry_wrapper = __esm({
|
|
|
41858
42081
|
try {
|
|
41859
42082
|
return await toRetry();
|
|
41860
42083
|
} catch (e6) {
|
|
41861
|
-
await new Promise((
|
|
42084
|
+
await new Promise((resolve17) => setTimeout(resolve17, delayMs));
|
|
41862
42085
|
}
|
|
41863
42086
|
}
|
|
41864
42087
|
return await toRetry();
|
|
@@ -47093,14 +47316,14 @@ var init_readableStreamHasher = __esm({
|
|
|
47093
47316
|
const hash = new hashCtor();
|
|
47094
47317
|
const hashCalculator = new HashCalculator(hash);
|
|
47095
47318
|
readableStream.pipe(hashCalculator);
|
|
47096
|
-
return new Promise((
|
|
47319
|
+
return new Promise((resolve17, reject) => {
|
|
47097
47320
|
readableStream.on("error", (err) => {
|
|
47098
47321
|
hashCalculator.end();
|
|
47099
47322
|
reject(err);
|
|
47100
47323
|
});
|
|
47101
47324
|
hashCalculator.on("error", reject);
|
|
47102
47325
|
hashCalculator.on("finish", () => {
|
|
47103
|
-
hash.digest().then(
|
|
47326
|
+
hash.digest().then(resolve17).catch(reject);
|
|
47104
47327
|
});
|
|
47105
47328
|
});
|
|
47106
47329
|
};
|
|
@@ -52140,7 +52363,7 @@ var init_table_policy = __esm({
|
|
|
52140
52363
|
});
|
|
52141
52364
|
|
|
52142
52365
|
// src/ai/llm-client.ts
|
|
52143
|
-
import { createRequire as
|
|
52366
|
+
import { createRequire as createRequire3 } from "module";
|
|
52144
52367
|
var DEFAULT_MODEL, CHEAPEST_MODEL;
|
|
52145
52368
|
var init_llm_client = __esm({
|
|
52146
52369
|
"src/ai/llm-client.ts"() {
|
|
@@ -52314,7 +52537,7 @@ var init_summarize = __esm({
|
|
|
52314
52537
|
import { JSDOM } from "jsdom";
|
|
52315
52538
|
import { Readability } from "@mozilla/readability";
|
|
52316
52539
|
import { basename as basename5 } from "path";
|
|
52317
|
-
import { createRequire as
|
|
52540
|
+
import { createRequire as createRequire4 } from "module";
|
|
52318
52541
|
async function crawlUrl(rawUrl, opts = {}) {
|
|
52319
52542
|
const u2 = await assertSafeUrl(rawUrl, opts.allowPrivate ?? false);
|
|
52320
52543
|
const fetchImpl = opts.fetcher ?? fetch;
|
|
@@ -52561,7 +52784,7 @@ async function renderViaPlaywright(url, timeoutMs, warnIfMissing = false) {
|
|
|
52561
52784
|
let chromium;
|
|
52562
52785
|
try {
|
|
52563
52786
|
const importMetaUrl = import.meta.url;
|
|
52564
|
-
const req = importMetaUrl ?
|
|
52787
|
+
const req = importMetaUrl ? createRequire4(importMetaUrl) : __require;
|
|
52565
52788
|
const pw = req("playwright");
|
|
52566
52789
|
chromium = pw.chromium;
|
|
52567
52790
|
} catch {
|
|
@@ -52618,7 +52841,7 @@ function parsePageParam(raw, kind) {
|
|
|
52618
52841
|
}
|
|
52619
52842
|
function readJson(req, opts = {}) {
|
|
52620
52843
|
const maxBytes = opts.maxBytes ?? DEFAULT_BODY_MAX_BYTES;
|
|
52621
|
-
return new Promise((
|
|
52844
|
+
return new Promise((resolve17, reject) => {
|
|
52622
52845
|
let raw = "";
|
|
52623
52846
|
let overflowed = false;
|
|
52624
52847
|
req.setEncoding("utf8");
|
|
@@ -52634,7 +52857,7 @@ function readJson(req, opts = {}) {
|
|
|
52634
52857
|
req.on("end", () => {
|
|
52635
52858
|
if (overflowed) return;
|
|
52636
52859
|
try {
|
|
52637
|
-
|
|
52860
|
+
resolve17(raw ? JSON.parse(raw) : {});
|
|
52638
52861
|
} catch {
|
|
52639
52862
|
reject(new Error("Invalid JSON body"));
|
|
52640
52863
|
}
|
|
@@ -52654,11 +52877,12 @@ ${err.stack ?? ""}`);
|
|
|
52654
52877
|
sendJson(res, { error: err.message }, status);
|
|
52655
52878
|
}
|
|
52656
52879
|
}
|
|
52657
|
-
var DEFAULT_BODY_MAX_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
52880
|
+
var DEFAULT_BODY_MAX_BYTES, MAX_INGEST_BYTES, MAX_ROWS_PAGE, DEFAULT_ROWS_PAGE, BodyTooLargeError;
|
|
52658
52881
|
var init_http2 = __esm({
|
|
52659
52882
|
"src/gui/http.ts"() {
|
|
52660
52883
|
"use strict";
|
|
52661
52884
|
DEFAULT_BODY_MAX_BYTES = 1e6;
|
|
52885
|
+
MAX_INGEST_BYTES = 5e7;
|
|
52662
52886
|
MAX_ROWS_PAGE = 1e3;
|
|
52663
52887
|
DEFAULT_ROWS_PAGE = 500;
|
|
52664
52888
|
BodyTooLargeError = class extends Error {
|
|
@@ -53651,7 +53875,7 @@ var init_oauth = __esm({
|
|
|
53651
53875
|
|
|
53652
53876
|
// src/gui/assistant-routes.ts
|
|
53653
53877
|
function readBuffer(req, maxBytes = 25e6) {
|
|
53654
|
-
return new Promise((
|
|
53878
|
+
return new Promise((resolve17, reject) => {
|
|
53655
53879
|
const chunks = [];
|
|
53656
53880
|
let size = 0;
|
|
53657
53881
|
req.on("data", (c6) => {
|
|
@@ -53660,7 +53884,7 @@ function readBuffer(req, maxBytes = 25e6) {
|
|
|
53660
53884
|
else chunks.push(c6);
|
|
53661
53885
|
});
|
|
53662
53886
|
req.on("end", () => {
|
|
53663
|
-
|
|
53887
|
+
resolve17(Buffer.concat(chunks));
|
|
53664
53888
|
});
|
|
53665
53889
|
req.on("error", reject);
|
|
53666
53890
|
});
|
|
@@ -54961,7 +55185,7 @@ async function takeHostSlot(host, minIntervalMs = urlIngestConfig().hostMinInter
|
|
|
54961
55185
|
const earliest = Math.max(now2, hostNextAllowed.get(key) ?? 0);
|
|
54962
55186
|
hostNextAllowed.set(key, earliest + minIntervalMs);
|
|
54963
55187
|
const wait = earliest - now2;
|
|
54964
|
-
if (wait > 0) await new Promise((
|
|
55188
|
+
if (wait > 0) await new Promise((resolve17) => setTimeout(resolve17, wait));
|
|
54965
55189
|
}
|
|
54966
55190
|
var Semaphore, FetchBudget, sharedGate, hostNextAllowed;
|
|
54967
55191
|
var init_fetch_policy = __esm({
|
|
@@ -54977,7 +55201,7 @@ var init_fetch_policy = __esm({
|
|
|
54977
55201
|
if (this.permits > 0) {
|
|
54978
55202
|
this.permits -= 1;
|
|
54979
55203
|
} else {
|
|
54980
|
-
await new Promise((
|
|
55204
|
+
await new Promise((resolve17) => this.waiters.push(resolve17));
|
|
54981
55205
|
}
|
|
54982
55206
|
let released = false;
|
|
54983
55207
|
return () => {
|
|
@@ -55214,8 +55438,8 @@ function fileContentGroups(rows, fuzzy, threshold) {
|
|
|
55214
55438
|
const t8 = get2(r6, "extracted_text");
|
|
55215
55439
|
return typeof t8 === "string" && t8.trim().length > 0;
|
|
55216
55440
|
}).map((r6) => {
|
|
55217
|
-
const
|
|
55218
|
-
const key = fuzzy ? "txt:" +
|
|
55441
|
+
const norm3 = normalizeText(get2(r6, "extracted_text"));
|
|
55442
|
+
const key = fuzzy ? "txt:" + norm3.slice(0, 2e3) : "txt:" + createHash11("sha256").update(norm3).digest("hex");
|
|
55219
55443
|
return { id: String(get2(r6, "id")), key, createdAt: cellStrOrNull(get2(r6, "created_at")) };
|
|
55220
55444
|
});
|
|
55221
55445
|
const txtGroups = findDuplicateGroups(txtItems, {
|
|
@@ -55481,11 +55705,11 @@ function stripHtml(html) {
|
|
|
55481
55705
|
const text = decodeXmlEntities(stripTags(noScript));
|
|
55482
55706
|
return text.replace(/[ \t\f\r]+/g, " ").replace(/ *\n */g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
55483
55707
|
}
|
|
55484
|
-
async function unzip(
|
|
55708
|
+
async function unzip(path3) {
|
|
55485
55709
|
const fflate = await loadParser("fflate");
|
|
55486
55710
|
if (!fflate || typeof fflate.unzipSync !== "function") return null;
|
|
55487
55711
|
try {
|
|
55488
|
-
const buf = await readFile6(
|
|
55712
|
+
const buf = await readFile6(path3);
|
|
55489
55713
|
let total = 0;
|
|
55490
55714
|
return fflate.unzipSync(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength), {
|
|
55491
55715
|
filter: (file) => {
|
|
@@ -55514,35 +55738,35 @@ var init_helpers2 = __esm({
|
|
|
55514
55738
|
|
|
55515
55739
|
// src/gui/ai/doc/ooxml.ts
|
|
55516
55740
|
import { readFile as readFile7 } from "fs/promises";
|
|
55517
|
-
async function extractDocx(
|
|
55741
|
+
async function extractDocx(path3) {
|
|
55518
55742
|
const mod = await loadParser("mammoth");
|
|
55519
55743
|
const lib = mod?.default ?? mod;
|
|
55520
55744
|
if (!lib || typeof lib.extractRawText !== "function") return null;
|
|
55521
55745
|
try {
|
|
55522
|
-
const { value } = await lib.extractRawText({ path:
|
|
55746
|
+
const { value } = await lib.extractRawText({ path: path3 });
|
|
55523
55747
|
return nullIfEmpty(value);
|
|
55524
55748
|
} catch {
|
|
55525
55749
|
return null;
|
|
55526
55750
|
}
|
|
55527
55751
|
}
|
|
55528
|
-
async function extractDoc(
|
|
55752
|
+
async function extractDoc(path3) {
|
|
55529
55753
|
const mod = await loadParser(
|
|
55530
55754
|
"word-extractor"
|
|
55531
55755
|
);
|
|
55532
55756
|
const Ctor = mod && "default" in mod ? mod.default : mod;
|
|
55533
55757
|
if (typeof Ctor !== "function") return null;
|
|
55534
55758
|
try {
|
|
55535
|
-
const doc = await new Ctor().extract(
|
|
55759
|
+
const doc = await new Ctor().extract(path3);
|
|
55536
55760
|
return nullIfEmpty(doc.getBody());
|
|
55537
55761
|
} catch {
|
|
55538
55762
|
return null;
|
|
55539
55763
|
}
|
|
55540
55764
|
}
|
|
55541
|
-
async function extractPdf(
|
|
55765
|
+
async function extractPdf(path3) {
|
|
55542
55766
|
const unpdf = await loadParser("unpdf");
|
|
55543
55767
|
if (!unpdf || typeof unpdf.getDocumentProxy !== "function") return null;
|
|
55544
55768
|
try {
|
|
55545
|
-
const buf = await readFile7(
|
|
55769
|
+
const buf = await readFile7(path3);
|
|
55546
55770
|
const data = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
55547
55771
|
const text = await withTimeout(
|
|
55548
55772
|
(async () => {
|
|
@@ -55574,8 +55798,8 @@ function slideText(xml) {
|
|
|
55574
55798
|
}
|
|
55575
55799
|
return paras.join("\n");
|
|
55576
55800
|
}
|
|
55577
|
-
async function extractPptx(
|
|
55578
|
-
const entries = await unzip(
|
|
55801
|
+
async function extractPptx(path3) {
|
|
55802
|
+
const entries = await unzip(path3);
|
|
55579
55803
|
if (!entries) return null;
|
|
55580
55804
|
const slides = Object.keys(entries).filter((n3) => /^ppt\/slides\/slide\d+\.xml$/.test(n3)).sort((a6, b6) => partNumber(a6) - partNumber(b6));
|
|
55581
55805
|
if (slides.length === 0) return null;
|
|
@@ -55593,8 +55817,8 @@ async function extractPptx(path2) {
|
|
|
55593
55817
|
}
|
|
55594
55818
|
return nullIfEmpty(parts.join("\n\n"));
|
|
55595
55819
|
}
|
|
55596
|
-
async function extractXlsx(
|
|
55597
|
-
const entries = await unzip(
|
|
55820
|
+
async function extractXlsx(path3) {
|
|
55821
|
+
const entries = await unzip(path3);
|
|
55598
55822
|
if (!entries) return null;
|
|
55599
55823
|
const shared = [];
|
|
55600
55824
|
const ssBytes = entries["xl/sharedStrings.xml"];
|
|
@@ -55652,8 +55876,8 @@ function odfWhitespace(s2) {
|
|
|
55652
55876
|
function odfParagraph(inner) {
|
|
55653
55877
|
return decodeXmlEntities(stripTags(odfWhitespace(inner))).trim();
|
|
55654
55878
|
}
|
|
55655
|
-
async function extractOdfText(
|
|
55656
|
-
const entries = await unzip(
|
|
55879
|
+
async function extractOdfText(path3) {
|
|
55880
|
+
const entries = await unzip(path3);
|
|
55657
55881
|
if (!entries) return null;
|
|
55658
55882
|
const contentBytes = entries["content.xml"];
|
|
55659
55883
|
if (!contentBytes) return null;
|
|
@@ -55675,8 +55899,8 @@ async function extractOdfText(path2) {
|
|
|
55675
55899
|
}
|
|
55676
55900
|
return nullIfEmpty(lines.join("\n"));
|
|
55677
55901
|
}
|
|
55678
|
-
async function extractOds(
|
|
55679
|
-
const entries = await unzip(
|
|
55902
|
+
async function extractOds(path3) {
|
|
55903
|
+
const entries = await unzip(path3);
|
|
55680
55904
|
if (!entries) return null;
|
|
55681
55905
|
const contentBytes = entries["content.xml"];
|
|
55682
55906
|
if (!contentBytes) return null;
|
|
@@ -55735,8 +55959,8 @@ function resolveHref(baseDir, href) {
|
|
|
55735
55959
|
}
|
|
55736
55960
|
return normalizeZipPath(baseDir + h6);
|
|
55737
55961
|
}
|
|
55738
|
-
async function extractEpub(
|
|
55739
|
-
const entries = await unzip(
|
|
55962
|
+
async function extractEpub(path3) {
|
|
55963
|
+
const entries = await unzip(path3);
|
|
55740
55964
|
if (!entries) return null;
|
|
55741
55965
|
let order = [];
|
|
55742
55966
|
const container = entries["META-INF/container.xml"];
|
|
@@ -55820,9 +56044,9 @@ function rtfToText(rtf) {
|
|
|
55820
56044
|
s2 = s2.replace(/[{}]/g, "");
|
|
55821
56045
|
return s2.replace(/[ \t]+/g, (m4) => m4.includes(" ") ? " " : " ").replace(/[ \t]\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
55822
56046
|
}
|
|
55823
|
-
async function extractRtf(
|
|
56047
|
+
async function extractRtf(path3) {
|
|
55824
56048
|
try {
|
|
55825
|
-
const raw = await readFile8(
|
|
56049
|
+
const raw = await readFile8(path3, "latin1");
|
|
55826
56050
|
if (!raw.startsWith("{\\rtf")) return null;
|
|
55827
56051
|
return nullIfEmpty(rtfToText(raw));
|
|
55828
56052
|
} catch {
|
|
@@ -55868,27 +56092,27 @@ var init_other = __esm({
|
|
|
55868
56092
|
});
|
|
55869
56093
|
|
|
55870
56094
|
// src/gui/ai/doc-extractors.ts
|
|
55871
|
-
async function extractDocument(
|
|
56095
|
+
async function extractDocument(path3, ext) {
|
|
55872
56096
|
switch (ext) {
|
|
55873
56097
|
case ".docx":
|
|
55874
|
-
return extractDocx(
|
|
56098
|
+
return extractDocx(path3);
|
|
55875
56099
|
case ".doc":
|
|
55876
|
-
return extractDoc(
|
|
56100
|
+
return extractDoc(path3);
|
|
55877
56101
|
case ".pdf":
|
|
55878
|
-
return extractPdf(
|
|
56102
|
+
return extractPdf(path3);
|
|
55879
56103
|
case ".pptx":
|
|
55880
|
-
return extractPptx(
|
|
56104
|
+
return extractPptx(path3);
|
|
55881
56105
|
case ".xlsx":
|
|
55882
|
-
return extractXlsx(
|
|
56106
|
+
return extractXlsx(path3);
|
|
55883
56107
|
case ".odt":
|
|
55884
56108
|
case ".odp":
|
|
55885
|
-
return extractOdfText(
|
|
56109
|
+
return extractOdfText(path3);
|
|
55886
56110
|
case ".ods":
|
|
55887
|
-
return extractOds(
|
|
56111
|
+
return extractOds(path3);
|
|
55888
56112
|
case ".epub":
|
|
55889
|
-
return extractEpub(
|
|
56113
|
+
return extractEpub(path3);
|
|
55890
56114
|
case ".rtf":
|
|
55891
|
-
return extractRtf(
|
|
56115
|
+
return extractRtf(path3);
|
|
55892
56116
|
default:
|
|
55893
56117
|
return null;
|
|
55894
56118
|
}
|
|
@@ -55911,17 +56135,17 @@ function languageOf(name) {
|
|
|
55911
56135
|
function truncate2(s2) {
|
|
55912
56136
|
return s2.length > MAX_TEXT2 ? s2.slice(0, MAX_TEXT2) : s2;
|
|
55913
56137
|
}
|
|
55914
|
-
async function parseFile(
|
|
55915
|
-
const name = originalName ?? basename7(
|
|
56138
|
+
async function parseFile(path3, mimeHint, originalName) {
|
|
56139
|
+
const name = originalName ?? basename7(path3);
|
|
55916
56140
|
const ext = extname(name).toLowerCase();
|
|
55917
56141
|
const lang = languageOf(name);
|
|
55918
56142
|
if (lang) {
|
|
55919
|
-
return { text: truncate2(await readFile9(
|
|
56143
|
+
return { text: truncate2(await readFile9(path3, "utf8")), language: lang };
|
|
55920
56144
|
}
|
|
55921
56145
|
if (mimeHint && TEXT_MIME.test(mimeHint) || TEXT_EXT.has(ext)) {
|
|
55922
|
-
return { text: truncate2(await readFile9(
|
|
56146
|
+
return { text: truncate2(await readFile9(path3, "utf8")) };
|
|
55923
56147
|
}
|
|
55924
|
-
const doc = await extractDocument(
|
|
56148
|
+
const doc = await extractDocument(path3, ext);
|
|
55925
56149
|
if (doc != null) {
|
|
55926
56150
|
return { text: truncate2(doc) };
|
|
55927
56151
|
}
|
|
@@ -56510,8 +56734,8 @@ function isWriteConflict(e6) {
|
|
|
56510
56734
|
function normalizeUrl(s2) {
|
|
56511
56735
|
try {
|
|
56512
56736
|
const u2 = new URL(s2.trim());
|
|
56513
|
-
const
|
|
56514
|
-
return `${u2.protocol}//${u2.host.toLowerCase()}${
|
|
56737
|
+
const path3 = u2.pathname.replace(/\/+$/, "");
|
|
56738
|
+
return `${u2.protocol}//${u2.host.toLowerCase()}${path3}${u2.search}`;
|
|
56515
56739
|
} catch {
|
|
56516
56740
|
return null;
|
|
56517
56741
|
}
|
|
@@ -57235,7 +57459,7 @@ var init_tools = __esm({
|
|
|
57235
57459
|
});
|
|
57236
57460
|
|
|
57237
57461
|
// src/gui/ai/chat.ts
|
|
57238
|
-
import { createRequire as
|
|
57462
|
+
import { createRequire as createRequire7 } from "module";
|
|
57239
57463
|
function capToolResult(s2) {
|
|
57240
57464
|
if (s2.length <= MAX_TOOL_RESULT_CHARS) return s2;
|
|
57241
57465
|
if (s2.length > MAX_TOOL_RESULT_SKIP)
|
|
@@ -57468,7 +57692,7 @@ async function* runChat(opts) {
|
|
|
57468
57692
|
function loadSdk() {
|
|
57469
57693
|
if (!_sdk) {
|
|
57470
57694
|
const importMetaUrl = import.meta.url;
|
|
57471
|
-
const req = importMetaUrl ?
|
|
57695
|
+
const req = importMetaUrl ? createRequire7(importMetaUrl) : __require;
|
|
57472
57696
|
try {
|
|
57473
57697
|
_sdk = req("@anthropic-ai/sdk");
|
|
57474
57698
|
} catch (err) {
|
|
@@ -58123,8 +58347,8 @@ function isRetryableDbError(err) {
|
|
|
58123
58347
|
return msg.includes("database is locked") || msg.includes("connection terminated") || msg.includes("connection reset") || msg.includes("server closed the connection");
|
|
58124
58348
|
}
|
|
58125
58349
|
var retryDepth = new AsyncLocalStorage();
|
|
58126
|
-
var defaultSleep = (ms) => new Promise((
|
|
58127
|
-
setTimeout(
|
|
58350
|
+
var defaultSleep = (ms) => new Promise((resolve17) => {
|
|
58351
|
+
setTimeout(resolve17, ms);
|
|
58128
58352
|
});
|
|
58129
58353
|
async function withRetry(fn, opts = {}) {
|
|
58130
58354
|
if (retryDepth.getStore()) return fn();
|
|
@@ -58323,13 +58547,13 @@ import { createHash as createHash4 } from "crypto";
|
|
|
58323
58547
|
import { createReadStream, existsSync as existsSync12, mkdirSync as mkdirSync6, statSync as statSync5, copyFileSync as copyFileSync3 } from "fs";
|
|
58324
58548
|
import { basename as basename3, join as join12 } from "path";
|
|
58325
58549
|
async function hashFile(srcPath) {
|
|
58326
|
-
return new Promise((
|
|
58550
|
+
return new Promise((resolve17, reject) => {
|
|
58327
58551
|
const hash = createHash4("sha256");
|
|
58328
58552
|
const stream = createReadStream(srcPath);
|
|
58329
58553
|
stream.on("data", (chunk) => hash.update(chunk));
|
|
58330
58554
|
stream.on("error", reject);
|
|
58331
58555
|
stream.on("end", () => {
|
|
58332
|
-
|
|
58556
|
+
resolve17(hash.digest("hex"));
|
|
58333
58557
|
});
|
|
58334
58558
|
});
|
|
58335
58559
|
}
|
|
@@ -58587,20 +58811,20 @@ function blobHandle(row, latticeRoot) {
|
|
|
58587
58811
|
};
|
|
58588
58812
|
}
|
|
58589
58813
|
function fsHandle(row) {
|
|
58590
|
-
const
|
|
58814
|
+
const path3 = row.ref_uri ?? "";
|
|
58591
58815
|
return {
|
|
58592
58816
|
kind: "local_ref",
|
|
58593
58817
|
provider: "fs",
|
|
58594
|
-
location:
|
|
58818
|
+
location: path3,
|
|
58595
58819
|
async readContent() {
|
|
58596
58820
|
try {
|
|
58597
|
-
return await readFile4(
|
|
58821
|
+
return await readFile4(path3);
|
|
58598
58822
|
} catch (e6) {
|
|
58599
|
-
throw new ReferenceUnavailableError(
|
|
58823
|
+
throw new ReferenceUnavailableError(path3, e6.message);
|
|
58600
58824
|
}
|
|
58601
58825
|
},
|
|
58602
58826
|
async getMetadata() {
|
|
58603
|
-
return statMeta(
|
|
58827
|
+
return statMeta(path3, row);
|
|
58604
58828
|
}
|
|
58605
58829
|
};
|
|
58606
58830
|
}
|
|
@@ -58669,9 +58893,9 @@ function meta(available, parts = {}) {
|
|
|
58669
58893
|
if (parts.extra != null) m4.extra = parts.extra;
|
|
58670
58894
|
return m4;
|
|
58671
58895
|
}
|
|
58672
|
-
async function statMeta(
|
|
58896
|
+
async function statMeta(path3, row) {
|
|
58673
58897
|
try {
|
|
58674
|
-
const s2 = await stat(
|
|
58898
|
+
const s2 = await stat(path3);
|
|
58675
58899
|
return meta(true, {
|
|
58676
58900
|
size_bytes: s2.size,
|
|
58677
58901
|
modified_at: s2.mtime.toISOString(),
|
|
@@ -59406,12 +59630,12 @@ function isBetter(next, prev) {
|
|
|
59406
59630
|
|
|
59407
59631
|
// src/ai/vision.ts
|
|
59408
59632
|
init_llm_client();
|
|
59409
|
-
import { createRequire as
|
|
59633
|
+
import { createRequire as createRequire5 } from "module";
|
|
59410
59634
|
import { readFile as readFile5 } from "fs/promises";
|
|
59411
59635
|
var DEFAULT_PROMPT = "Describe this image for a knowledge base in 2-4 factual sentences: what it shows, any visible text, and notable details. No preamble.";
|
|
59412
59636
|
var MAX_DIM = 1568;
|
|
59413
|
-
async function describeImage(auth,
|
|
59414
|
-
const data = (await normalizeImage(
|
|
59637
|
+
async function describeImage(auth, path3, opts = {}) {
|
|
59638
|
+
const data = (await normalizeImage(path3, opts.maxBytes ?? 14e5)).toString("base64");
|
|
59415
59639
|
const sender = opts.sender ?? defaultSender(auth);
|
|
59416
59640
|
const text = await sender({
|
|
59417
59641
|
media_type: "image/jpeg",
|
|
@@ -59422,8 +59646,8 @@ async function describeImage(auth, path2, opts = {}) {
|
|
|
59422
59646
|
return text.trim();
|
|
59423
59647
|
}
|
|
59424
59648
|
var DEFAULT_PDF_PROMPT = "Read this document for a knowledge base. First transcribe its readable text, then add a 2-4 sentence factual summary of what it is and its key details. It may be a scanned/image-only PDF \u2014 read the text from the page images. No preamble.";
|
|
59425
|
-
async function describePdf(auth,
|
|
59426
|
-
const buf = await readFile5(
|
|
59649
|
+
async function describePdf(auth, path3, opts = {}) {
|
|
59650
|
+
const buf = await readFile5(path3);
|
|
59427
59651
|
const maxBytes = opts.maxBytes ?? 3e7;
|
|
59428
59652
|
if (buf.length > maxBytes) {
|
|
59429
59653
|
throw new Error(
|
|
@@ -59438,19 +59662,19 @@ async function describePdf(auth, path2, opts = {}) {
|
|
|
59438
59662
|
});
|
|
59439
59663
|
return text.trim();
|
|
59440
59664
|
}
|
|
59441
|
-
async function normalizeImage(
|
|
59665
|
+
async function normalizeImage(path3, maxBytes) {
|
|
59442
59666
|
const sharpMod = await import("sharp");
|
|
59443
59667
|
const sharp = sharpMod.default;
|
|
59444
59668
|
let quality = 80;
|
|
59445
|
-
let buf = await renderJpeg(sharp,
|
|
59669
|
+
let buf = await renderJpeg(sharp, path3, quality);
|
|
59446
59670
|
while (buf.length > maxBytes && quality > 35) {
|
|
59447
59671
|
quality -= 15;
|
|
59448
|
-
buf = await renderJpeg(sharp,
|
|
59672
|
+
buf = await renderJpeg(sharp, path3, quality);
|
|
59449
59673
|
}
|
|
59450
59674
|
return buf;
|
|
59451
59675
|
}
|
|
59452
|
-
function renderJpeg(sharp,
|
|
59453
|
-
return sharp(
|
|
59676
|
+
function renderJpeg(sharp, path3, quality) {
|
|
59677
|
+
return sharp(path3).rotate().resize({ width: MAX_DIM, height: MAX_DIM, fit: "inside", withoutEnlargement: true }).jpeg({ quality }).toBuffer();
|
|
59454
59678
|
}
|
|
59455
59679
|
function buildVisionAnthropicConfig(auth) {
|
|
59456
59680
|
const config = {};
|
|
@@ -59468,7 +59692,7 @@ function buildVisionAnthropicConfig(auth) {
|
|
|
59468
59692
|
function defaultSender(auth) {
|
|
59469
59693
|
return async (input) => {
|
|
59470
59694
|
const importMetaUrl = import.meta.url;
|
|
59471
|
-
const req = importMetaUrl ?
|
|
59695
|
+
const req = importMetaUrl ? createRequire5(importMetaUrl) : __require;
|
|
59472
59696
|
const sdk = req("@anthropic-ai/sdk");
|
|
59473
59697
|
const Anthropic = sdk.Anthropic ?? sdk.default;
|
|
59474
59698
|
if (!Anthropic) throw new Error("Could not resolve Anthropic from '@anthropic-ai/sdk'");
|
|
@@ -59495,7 +59719,7 @@ function defaultSender(auth) {
|
|
|
59495
59719
|
function defaultPdfSender(auth) {
|
|
59496
59720
|
return async (input) => {
|
|
59497
59721
|
const importMetaUrl = import.meta.url;
|
|
59498
|
-
const req = importMetaUrl ?
|
|
59722
|
+
const req = importMetaUrl ? createRequire5(importMetaUrl) : __require;
|
|
59499
59723
|
const sdk = req("@anthropic-ai/sdk");
|
|
59500
59724
|
const Anthropic = sdk.Anthropic ?? sdk.default;
|
|
59501
59725
|
if (!Anthropic) throw new Error("Could not resolve Anthropic from '@anthropic-ai/sdk'");
|
|
@@ -59528,7 +59752,7 @@ init_http2();
|
|
|
59528
59752
|
import { createServer } from "http";
|
|
59529
59753
|
import { spawn as spawn2 } from "child_process";
|
|
59530
59754
|
import { WebSocketServer, WebSocket } from "ws";
|
|
59531
|
-
import { dirname as dirname16, resolve as
|
|
59755
|
+
import { dirname as dirname16, resolve as resolve15 } from "path";
|
|
59532
59756
|
|
|
59533
59757
|
// src/gui/active-db.ts
|
|
59534
59758
|
init_adapter();
|
|
@@ -59536,9 +59760,17 @@ init_members();
|
|
|
59536
59760
|
init_native_entities();
|
|
59537
59761
|
async function changeVisibleToActiveRole(db, payload) {
|
|
59538
59762
|
if (db.getDialect() !== "postgres") return true;
|
|
59539
|
-
if (payload.op === "delete" || payload.op === "DELETE") return true;
|
|
59540
59763
|
if (!payload.table_name || !payload.pk) return false;
|
|
59541
59764
|
try {
|
|
59765
|
+
if (isDeleteOp(payload.op)) {
|
|
59766
|
+
if (payload.del_owner_role == null) return false;
|
|
59767
|
+
const row2 = await getAsyncOrSync(
|
|
59768
|
+
db.adapter,
|
|
59769
|
+
`SELECT lattice_delete_visible(?, ?, ?::text[]) AS v`,
|
|
59770
|
+
[payload.del_owner_role, payload.del_visibility ?? null, payload.del_grantees ?? []]
|
|
59771
|
+
);
|
|
59772
|
+
return row2?.v === true || row2?.v === "t" || row2?.v === 1;
|
|
59773
|
+
}
|
|
59542
59774
|
const row = await getAsyncOrSync(db.adapter, `SELECT lattice_row_visible(?, ?) AS v`, [
|
|
59543
59775
|
payload.table_name,
|
|
59544
59776
|
payload.pk
|
|
@@ -59798,12 +60030,12 @@ init_postgres();
|
|
|
59798
60030
|
|
|
59799
60031
|
// src/gui/realtime.ts
|
|
59800
60032
|
import { EventEmitter } from "events";
|
|
59801
|
-
import { createRequire as
|
|
60033
|
+
import { createRequire as createRequire6 } from "module";
|
|
59802
60034
|
var _pgModule = null;
|
|
59803
60035
|
function loadPg() {
|
|
59804
60036
|
if (_pgModule) return _pgModule;
|
|
59805
60037
|
const importMetaUrl = import.meta.url;
|
|
59806
|
-
const requireFromHere = importMetaUrl ?
|
|
60038
|
+
const requireFromHere = importMetaUrl ? createRequire6(importMetaUrl) : (
|
|
59807
60039
|
// CJS fallback — Node provides `require` on every CJS module scope.
|
|
59808
60040
|
__require
|
|
59809
60041
|
);
|
|
@@ -60029,9 +60261,9 @@ var RealtimeBroker = class {
|
|
|
60029
60261
|
() => "ended"
|
|
60030
60262
|
// a graceful-close error is still "closed enough"
|
|
60031
60263
|
);
|
|
60032
|
-
const timedOut = new Promise((
|
|
60264
|
+
const timedOut = new Promise((resolve17) => {
|
|
60033
60265
|
timer = setTimeout(() => {
|
|
60034
|
-
|
|
60266
|
+
resolve17("timeout");
|
|
60035
60267
|
}, this.stopEndTimeoutMs);
|
|
60036
60268
|
timer.unref?.();
|
|
60037
60269
|
});
|
|
@@ -60076,7 +60308,10 @@ function parsePayload(raw) {
|
|
|
60076
60308
|
pk: typeof obj2.pk === "string" ? obj2.pk : null,
|
|
60077
60309
|
op: obj2.op,
|
|
60078
60310
|
owner_role: typeof obj2.owner_role === "string" ? obj2.owner_role : null,
|
|
60079
|
-
created_at: typeof obj2.created_at === "string" ? obj2.created_at : ""
|
|
60311
|
+
created_at: typeof obj2.created_at === "string" ? obj2.created_at : "",
|
|
60312
|
+
del_owner_role: typeof obj2.del_owner_role === "string" ? obj2.del_owner_role : null,
|
|
60313
|
+
del_visibility: typeof obj2.del_visibility === "string" ? obj2.del_visibility : null,
|
|
60314
|
+
del_grantees: Array.isArray(obj2.del_grantees) ? obj2.del_grantees.filter((g6) => typeof g6 === "string") : null
|
|
60080
60315
|
};
|
|
60081
60316
|
} catch {
|
|
60082
60317
|
return null;
|
|
@@ -61057,9 +61292,9 @@ function startBackgroundRender(active) {
|
|
|
61057
61292
|
}
|
|
61058
61293
|
function settleWithin(p3, ms) {
|
|
61059
61294
|
let timer;
|
|
61060
|
-
const timeout = new Promise((
|
|
61295
|
+
const timeout = new Promise((resolve17) => {
|
|
61061
61296
|
timer = setTimeout(() => {
|
|
61062
|
-
|
|
61297
|
+
resolve17("timeout");
|
|
61063
61298
|
}, ms);
|
|
61064
61299
|
timer.unref?.();
|
|
61065
61300
|
});
|
|
@@ -61114,9 +61349,9 @@ var SWITCH_OPEN_TIMEOUT_MS = 2e4;
|
|
|
61114
61349
|
async function openWithinTimeout(open, timeoutMs = SWITCH_OPEN_TIMEOUT_MS, dispose = disposeActive) {
|
|
61115
61350
|
const opening = open();
|
|
61116
61351
|
let timer;
|
|
61117
|
-
const timedOut = new Promise((
|
|
61352
|
+
const timedOut = new Promise((resolve17) => {
|
|
61118
61353
|
timer = setTimeout(() => {
|
|
61119
|
-
|
|
61354
|
+
resolve17("timeout");
|
|
61120
61355
|
}, timeoutMs);
|
|
61121
61356
|
timer.unref?.();
|
|
61122
61357
|
});
|
|
@@ -61174,7 +61409,7 @@ async function applySchemaConfig(active, entry, direction, autoRender) {
|
|
|
61174
61409
|
const doc = loadConfigDoc(active.configPath);
|
|
61175
61410
|
const inv = direction === "inverse";
|
|
61176
61411
|
const ddl = [];
|
|
61177
|
-
const has = (
|
|
61412
|
+
const has = (path3) => doc.getIn(path3) !== void 0;
|
|
61178
61413
|
const reAddEntity = async (name, def) => {
|
|
61179
61414
|
if (has(["entities", name])) {
|
|
61180
61415
|
throw new Error(`Cannot restore "${name}": an entity with that name already exists`);
|
|
@@ -62787,6 +63022,65 @@ var chatCss = ` /* \u2500\u2500 Chat bubbles + tool pills \u2500\u2500\u2500\
|
|
|
62787
63022
|
}
|
|
62788
63023
|
`;
|
|
62789
63024
|
|
|
63025
|
+
// src/gui/app/styles/inline-import.ts
|
|
63026
|
+
var inlineImportCss = `
|
|
63027
|
+
/* \u2500\u2500 Inline import confirm card (assistant rail) \u2500\u2500 */
|
|
63028
|
+
.cd-sub { margin: 10px 0 6px; font-size: 12px; color: var(--text-muted, #9aa3ad); }
|
|
63029
|
+
.cd-row { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; margin-top: 8px; }
|
|
63030
|
+
.cd-path {
|
|
63031
|
+
flex: 1 1 220px; min-width: 0; box-sizing: border-box; height: 34px; padding: 0 10px;
|
|
63032
|
+
border-radius: 6px; border: 1px solid #2a2f36;
|
|
63033
|
+
background: var(--panel, #0e1116); color: var(--text, #e6e8eb); font-size: 13px;
|
|
63034
|
+
}
|
|
63035
|
+
.cd-status { margin-top: 12px; font-size: 13px; line-height: 1.5; }
|
|
63036
|
+
.cd-status.ok { color: #bef264; }
|
|
63037
|
+
.cd-status.err { color: #f87171; }
|
|
63038
|
+
.cd-status a { color: var(--accent, #bef264); }
|
|
63039
|
+
.cd-btn {
|
|
63040
|
+
height: 34px; padding: 0 14px; border-radius: 6px; border: 1px solid #2a2f36;
|
|
63041
|
+
background: transparent; color: var(--text, #e6e8eb); font-size: 13px;
|
|
63042
|
+
font-weight: 600; cursor: pointer;
|
|
63043
|
+
}
|
|
63044
|
+
.cd-btn:hover { background: rgba(255, 255, 255, 0.06); }
|
|
63045
|
+
.cd-btn.cd-primary { background: #bef264; color: #0b0d10; border-color: #bef264; }
|
|
63046
|
+
.cd-btn.cd-primary:hover { filter: brightness(1.06); }
|
|
63047
|
+
.cd-import-list { margin: 10px 0 0; padding-left: 18px; font-size: 13px; line-height: 1.6; }
|
|
63048
|
+
.cd-import-list li { margin: 2px 0; }
|
|
63049
|
+
.imp-sub { margin: 16px 0 6px; font-size: 13px; color: var(--text, #e6e8eb); }
|
|
63050
|
+
.imp-modes { display: flex; flex-direction: column; gap: 8px; margin: 0 0 6px; }
|
|
63051
|
+
.imp-modes label {
|
|
63052
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
63053
|
+
padding: 8px 10px; border: 1px solid #2a2f36; border-radius: 6px; cursor: pointer;
|
|
63054
|
+
}
|
|
63055
|
+
.imp-modes label:hover { background: rgba(255, 255, 255, 0.04); }
|
|
63056
|
+
.imp-modes input { margin-top: 2px; }
|
|
63057
|
+
.imp-modes b { color: var(--text, #e6e8eb); }
|
|
63058
|
+
.imp-percol {
|
|
63059
|
+
display: flex; gap: 8px; align-items: flex-start; font-size: 13px; line-height: 1.4;
|
|
63060
|
+
margin: 8px 0 0; cursor: pointer; color: var(--text-dim, #aeb6c2);
|
|
63061
|
+
}
|
|
63062
|
+
.imp-percol input { margin-top: 2px; }
|
|
63063
|
+
.imp-match { border-left: 3px solid var(--accent, #7dd3fc); font-weight: 500; }
|
|
63064
|
+
.feed-item.import-confirm .imp-confirm-body { margin-top: 4px; }
|
|
63065
|
+
|
|
63066
|
+
/* \u2500\u2500 Live import progress in the card's log \u2500\u2500 */
|
|
63067
|
+
.feed-item.import-confirm .imp-card-log,
|
|
63068
|
+
.feed-item.import-live .imp-card-log {
|
|
63069
|
+
margin-top: 4px;
|
|
63070
|
+
font: 12px/1.6 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
|
63071
|
+
max-height: 200px; overflow-y: auto; color: var(--text-muted, #9aa3ad);
|
|
63072
|
+
}
|
|
63073
|
+
.imp-card-line { white-space: pre-wrap; word-break: break-word; }
|
|
63074
|
+
.imp-card-line.imp-done { color: var(--accent, #bef264); }
|
|
63075
|
+
.imp-card-line.imp-err { color: #f87171; }
|
|
63076
|
+
.imp-card-line.imp-spin::after {
|
|
63077
|
+
content: ''; display: inline-block; width: 10px; height: 10px; margin-left: 7px;
|
|
63078
|
+
border: 2px solid currentColor; border-right-color: transparent; border-radius: 50%;
|
|
63079
|
+
vertical-align: -1px; animation: imp-spin-kf 0.7s linear infinite;
|
|
63080
|
+
}
|
|
63081
|
+
@keyframes imp-spin-kf { to { transform: rotate(360deg); } }
|
|
63082
|
+
`;
|
|
63083
|
+
|
|
62790
63084
|
// src/gui/app/styles/index.ts
|
|
62791
63085
|
var css = [
|
|
62792
63086
|
tokensCss,
|
|
@@ -62808,7 +63102,8 @@ var css = [
|
|
|
62808
63102
|
fsWorkspaceCss,
|
|
62809
63103
|
settingsDrawerCss,
|
|
62810
63104
|
assistantRailCss,
|
|
62811
|
-
chatCss
|
|
63105
|
+
chatCss,
|
|
63106
|
+
inlineImportCss
|
|
62812
63107
|
].join("");
|
|
62813
63108
|
|
|
62814
63109
|
// src/gui/app/modules/display-config.ts
|
|
@@ -70379,6 +70674,11 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
70379
70674
|
// survivor if it was a duplicate). Multi-file drops do not navigate.
|
|
70380
70675
|
if (files.length === 1) {
|
|
70381
70676
|
uploadFile(files[0]).then(function (j) {
|
|
70677
|
+
// A structured source the server flagged as confirmable comes back with
|
|
70678
|
+
// an autoImport proposal \u2014 render the inline confirm card instead of
|
|
70679
|
+
// navigating to the file record. A silent import (autoImport.imported,
|
|
70680
|
+
// no reason) or a plain file keeps the open-the-record behavior.
|
|
70681
|
+
if (j && j.autoImport && j.autoImport.reason) { renderInlineImportCard(j.autoImport); return; }
|
|
70382
70682
|
if (j && (j.duplicateOf || j.id)) openSearchHit('files', j.duplicateOf || j.id);
|
|
70383
70683
|
});
|
|
70384
70684
|
return;
|
|
@@ -70388,7 +70688,15 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
70388
70688
|
var bar = ingestProgress(files.length);
|
|
70389
70689
|
var thunks = [];
|
|
70390
70690
|
for (var i = 0; i < files.length; i++) {
|
|
70391
|
-
(function (f) {
|
|
70691
|
+
(function (f) {
|
|
70692
|
+
thunks.push(function () {
|
|
70693
|
+
return uploadFile(f).then(function (j) {
|
|
70694
|
+
// A structured source within a batch still gets its own inline
|
|
70695
|
+
// confirm card (the batch as a whole does not navigate).
|
|
70696
|
+
if (j && j.autoImport && j.autoImport.reason) renderInlineImportCard(j.autoImport);
|
|
70697
|
+
});
|
|
70698
|
+
});
|
|
70699
|
+
})(files[i]);
|
|
70392
70700
|
}
|
|
70393
70701
|
runIngestBatch(thunks, INGEST_MAX_CONCURRENCY, bar.update).then(bar.done);
|
|
70394
70702
|
}
|
|
@@ -70544,6 +70852,237 @@ var createDatabaseWizardJs = ` // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\
|
|
|
70544
70852
|
})();
|
|
70545
70853
|
`;
|
|
70546
70854
|
|
|
70855
|
+
// src/gui/app/modules/inline-import.ts
|
|
70856
|
+
var inlineImportJs = `
|
|
70857
|
+
// \u2500\u2500 Inline structured-source import (confirm card in the assistant rail) \u2500\u2500
|
|
70858
|
+
function iiRailFeed() { return document.getElementById('rail-feed'); }
|
|
70859
|
+
function iiRailEmptyGone() {
|
|
70860
|
+
var e = document.getElementById('rail-empty');
|
|
70861
|
+
if (e) e.parentNode && e.parentNode.removeChild(e);
|
|
70862
|
+
}
|
|
70863
|
+
|
|
70864
|
+
// Read a newline-delimited-JSON response body, invoking onEvent(obj) per line.
|
|
70865
|
+
// Self-contained on purpose \u2014 this segment must not depend on any other.
|
|
70866
|
+
function iiStreamNdjson(url, payload, onEvent) {
|
|
70867
|
+
fetch(url, {
|
|
70868
|
+
method: 'POST',
|
|
70869
|
+
headers: { 'content-type': 'application/json' },
|
|
70870
|
+
body: JSON.stringify(payload),
|
|
70871
|
+
}).then(function (res) {
|
|
70872
|
+
if (!res.body || !res.body.getReader) {
|
|
70873
|
+
return res.text().then(function (t) {
|
|
70874
|
+
t.split('\\n').forEach(function (line) {
|
|
70875
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
70876
|
+
});
|
|
70877
|
+
});
|
|
70878
|
+
}
|
|
70879
|
+
var reader = res.body.getReader();
|
|
70880
|
+
var dec = new TextDecoder();
|
|
70881
|
+
var buf = '';
|
|
70882
|
+
function pump() {
|
|
70883
|
+
return reader.read().then(function (chunk) {
|
|
70884
|
+
if (chunk.done) {
|
|
70885
|
+
if (buf.trim()) { try { onEvent(JSON.parse(buf)); } catch (e) { /* skip */ } }
|
|
70886
|
+
return;
|
|
70887
|
+
}
|
|
70888
|
+
buf += dec.decode(chunk.value, { stream: true });
|
|
70889
|
+
var idx;
|
|
70890
|
+
while ((idx = buf.indexOf('\\n')) >= 0) {
|
|
70891
|
+
var line = buf.slice(0, idx);
|
|
70892
|
+
buf = buf.slice(idx + 1);
|
|
70893
|
+
if (line.trim()) { try { onEvent(JSON.parse(line)); } catch (e) { /* skip */ } }
|
|
70894
|
+
}
|
|
70895
|
+
return pump();
|
|
70896
|
+
});
|
|
70897
|
+
}
|
|
70898
|
+
return pump();
|
|
70899
|
+
}).catch(function (err) {
|
|
70900
|
+
onEvent({ phase: 'error', message: err && err.message ? err.message : 'Request failed' });
|
|
70901
|
+
});
|
|
70902
|
+
}
|
|
70903
|
+
|
|
70904
|
+
// Render the confirm card for a structured drop the server flagged as
|
|
70905
|
+
// needing confirmation. autoImport is the upload response's proposal:
|
|
70906
|
+
// { reason, fileId, plan:{entities,dimensions,linkages}, views, asOf,
|
|
70907
|
+
// asOfCandidates, asOfColumns, schemaMatch, matchedCount, totalEntities }.
|
|
70908
|
+
function renderInlineImportCard(autoImport) {
|
|
70909
|
+
if (!autoImport || !autoImport.fileId) return;
|
|
70910
|
+
var plan = autoImport.plan || {};
|
|
70911
|
+
var ents = plan.entities || [];
|
|
70912
|
+
var dims = plan.dimensions || [];
|
|
70913
|
+
var links = plan.linkages || [];
|
|
70914
|
+
var views = autoImport.views || [];
|
|
70915
|
+
var candidates = autoImport.asOfCandidates || [];
|
|
70916
|
+
var asOfColumns = autoImport.asOfColumns || [];
|
|
70917
|
+
var schemaMatch = autoImport.schemaMatch || {};
|
|
70918
|
+
var headerText = autoImport.reason === 'needs-confirm'
|
|
70919
|
+
? 'Add a dated snapshot'
|
|
70920
|
+
: 'Import as a new dataset';
|
|
70921
|
+
|
|
70922
|
+
iiRailEmptyGone();
|
|
70923
|
+
var feedEl = iiRailFeed();
|
|
70924
|
+
var card = document.createElement('div');
|
|
70925
|
+
card.className = 'feed-item import-confirm';
|
|
70926
|
+
var icon = document.createElement('div');
|
|
70927
|
+
icon.className = 'feed-icon';
|
|
70928
|
+
icon.textContent = '\u2913';
|
|
70929
|
+
var bodyEl = document.createElement('div');
|
|
70930
|
+
bodyEl.className = 'feed-body';
|
|
70931
|
+
var title = document.createElement('div');
|
|
70932
|
+
title.className = 'feed-summary';
|
|
70933
|
+
title.textContent = headerText;
|
|
70934
|
+
bodyEl.appendChild(title);
|
|
70935
|
+
|
|
70936
|
+
var parts = [];
|
|
70937
|
+
if (schemaMatch.isKnownDocument) {
|
|
70938
|
+
parts.push('<div class="cd-status ok imp-match">Recognized as a new period of an existing document — ' +
|
|
70939
|
+
schemaMatch.matchedCount + ' of ' + schemaMatch.totalEntities +
|
|
70940
|
+
' tables match what you already imported. It will be added as a dated snapshot.</div>');
|
|
70941
|
+
}
|
|
70942
|
+
parts.push('<div class="cd-status ok">Found ' + ents.length + ' entities, ' + dims.length +
|
|
70943
|
+
' dimensions, ' + links.length + ' links' +
|
|
70944
|
+
(views.length ? ', ' + views.length + ' reconstructed views (no duplicated rows)' : '') +
|
|
70945
|
+
'.</div><ul class="cd-import-list">');
|
|
70946
|
+
ents.forEach(function (e) {
|
|
70947
|
+
parts.push('<li><b>' + escapeHtml(e.name) + '</b> — ' + e.rowCount + ' rows, ' +
|
|
70948
|
+
(e.columns ? e.columns.length : 0) + ' cols · ' +
|
|
70949
|
+
(e.naturalKey ? 'key ' + escapeHtml(e.naturalKey) : 'keyless') + '</li>');
|
|
70950
|
+
});
|
|
70951
|
+
dims.forEach(function (d) {
|
|
70952
|
+
parts.push('<li><b>' + escapeHtml(d.name) + '</b> (dimension) — ' + d.distinctValues + ' values</li>');
|
|
70953
|
+
});
|
|
70954
|
+
views.forEach(function (v) {
|
|
70955
|
+
parts.push('<li><b>' + escapeHtml(v.name) + '</b> (view of ' + escapeHtml(v.master) + ' where ' +
|
|
70956
|
+
escapeHtml(v.filterColumn) + ' = ' + escapeHtml(String(v.filterValue)) + ') — ' +
|
|
70957
|
+
v.matchedRows + ' rows, not duplicated</li>');
|
|
70958
|
+
});
|
|
70959
|
+
parts.push('</ul>');
|
|
70960
|
+
|
|
70961
|
+
parts.push('<h4 class="imp-sub">As of date</h4>');
|
|
70962
|
+
var best = candidates[0];
|
|
70963
|
+
parts.push('<p class="cd-sub">' +
|
|
70964
|
+
(best ? 'Detected from ' + escapeHtml(best.evidence) + ' — edit if wrong.'
|
|
70965
|
+
: 'No date found in the file or its name — set the snapshot date, or leave blank to import undated.') +
|
|
70966
|
+
' A newer file is kept as a separate dated snapshot beside the prior one.</p>');
|
|
70967
|
+
parts.push('<div class="cd-row"><input class="cd-path" id="ii-asof" type="date" value="' + escapeHtml(autoImport.asOf || '') + '" aria-label="As of date" /></div>');
|
|
70968
|
+
if (candidates.length > 1) {
|
|
70969
|
+
parts.push('<div class="cd-sub">Other candidates: ' + candidates.slice(1, 5).map(function (c) {
|
|
70970
|
+
return '<a href="#" class="ii-asof-alt" data-date="' + escapeHtml(c.date) + '" title="' + escapeHtml(c.evidence) + '">' + escapeHtml(c.date) + '</a>';
|
|
70971
|
+
}).join(', ') + '</div>');
|
|
70972
|
+
}
|
|
70973
|
+
if (asOfColumns.length) {
|
|
70974
|
+
var colOpts = asOfColumns.slice(0, 6).map(function (c) {
|
|
70975
|
+
return '<option value="' + escapeHtml(c.column) + '" title="' + escapeHtml(c.evidence) + '">' +
|
|
70976
|
+
escapeHtml(c.column) + ' (' + escapeHtml(c.entity) + ', ' + c.distinctDates +
|
|
70977
|
+
' date' + (c.distinctDates === 1 ? '' : 's') + ')</option>';
|
|
70978
|
+
}).join('');
|
|
70979
|
+
parts.push('<label class="imp-percol"><input type="checkbox" id="ii-asof-percol"> ' +
|
|
70980
|
+
'<span>Date varies per row — use a date column instead (one file, many periods)</span></label>');
|
|
70981
|
+
parts.push('<div class="cd-row" id="ii-asof-col-row" style="display:none"><select class="cd-path" id="ii-asof-col">' + colOpts + '</select></div>');
|
|
70982
|
+
}
|
|
70983
|
+
|
|
70984
|
+
parts.push('<h4 class="imp-sub">What should Lattice bring in?</h4>');
|
|
70985
|
+
parts.push('<div class="imp-modes">' +
|
|
70986
|
+
'<label><input type="radio" name="ii-mode" value="both" checked> <span><b>Data model + contents</b> \u2014 the schema, the taxonomy, and all the rows.</span></label>' +
|
|
70987
|
+
'<label><input type="radio" name="ii-mode" value="schema"> <span><b>Data model / schema only</b> \u2014 tables, dimension values, and views. No rows.</span></label>' +
|
|
70988
|
+
'<label><input type="radio" name="ii-mode" value="contents"> <span><b>Contents only</b> \u2014 the rows and their links, into tables that already exist.</span></label>' +
|
|
70989
|
+
'</div>');
|
|
70990
|
+
parts.push('<div class="cd-row"><button class="cd-btn cd-primary" id="ii-apply" type="button">Import into Lattice</button></div>');
|
|
70991
|
+
parts.push('<div class="imp-card-log" id="ii-log"></div>');
|
|
70992
|
+
|
|
70993
|
+
var content = document.createElement('div');
|
|
70994
|
+
content.className = 'imp-confirm-body';
|
|
70995
|
+
content.innerHTML = parts.join('');
|
|
70996
|
+
bodyEl.appendChild(content);
|
|
70997
|
+
card.appendChild(icon);
|
|
70998
|
+
card.appendChild(bodyEl);
|
|
70999
|
+
if (feedEl) { feedEl.appendChild(card); feedEl.scrollTop = feedEl.scrollHeight; }
|
|
71000
|
+
|
|
71001
|
+
content.querySelectorAll('.ii-asof-alt').forEach(function (a) {
|
|
71002
|
+
a.addEventListener('click', function (e) {
|
|
71003
|
+
e.preventDefault();
|
|
71004
|
+
var input = document.getElementById('ii-asof');
|
|
71005
|
+
if (input) input.value = a.getAttribute('data-date') || '';
|
|
71006
|
+
});
|
|
71007
|
+
});
|
|
71008
|
+
var perCol = document.getElementById('ii-asof-percol');
|
|
71009
|
+
if (perCol) perCol.addEventListener('change', function () {
|
|
71010
|
+
var row = document.getElementById('ii-asof-col-row');
|
|
71011
|
+
var dateEl = document.getElementById('ii-asof');
|
|
71012
|
+
if (row) row.style.display = perCol.checked ? '' : 'none';
|
|
71013
|
+
if (dateEl) dateEl.disabled = perCol.checked;
|
|
71014
|
+
});
|
|
71015
|
+
|
|
71016
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
71017
|
+
if (applyBtn) applyBtn.addEventListener('click', function () {
|
|
71018
|
+
runInlineImport(autoImport.fileId, title, content);
|
|
71019
|
+
});
|
|
71020
|
+
}
|
|
71021
|
+
|
|
71022
|
+
// POST the confirmed proposal to /api/import/apply and stream the pipeline
|
|
71023
|
+
// live into the card's log. On 'done' show a success summary + refresh the
|
|
71024
|
+
// Objects nav in place; on 'error' show the message.
|
|
71025
|
+
function runInlineImport(fileId, title, content) {
|
|
71026
|
+
var sel = content.querySelector('input[name="ii-mode"]:checked');
|
|
71027
|
+
var mode = sel ? sel.value : 'both';
|
|
71028
|
+
var asofEl = document.getElementById('ii-asof');
|
|
71029
|
+
var asOf = asofEl ? asofEl.value : '';
|
|
71030
|
+
var perColEl = document.getElementById('ii-asof-percol');
|
|
71031
|
+
var colSel = document.getElementById('ii-asof-col');
|
|
71032
|
+
var asOfColumn = (perColEl && perColEl.checked && colSel) ? colSel.value : '';
|
|
71033
|
+
var applyBtn = document.getElementById('ii-apply');
|
|
71034
|
+
if (applyBtn) applyBtn.disabled = true;
|
|
71035
|
+
|
|
71036
|
+
var feedEl = iiRailFeed();
|
|
71037
|
+
var log = document.getElementById('ii-log');
|
|
71038
|
+
function addLine(text, cls) {
|
|
71039
|
+
if (!log) return null;
|
|
71040
|
+
var d = document.createElement('div');
|
|
71041
|
+
d.className = 'imp-card-line' + (cls ? ' ' + cls : '');
|
|
71042
|
+
d.textContent = text;
|
|
71043
|
+
log.appendChild(d);
|
|
71044
|
+
while (log.childNodes.length > 60) log.removeChild(log.firstChild);
|
|
71045
|
+
log.scrollTop = log.scrollHeight;
|
|
71046
|
+
if (feedEl) feedEl.scrollTop = feedEl.scrollHeight;
|
|
71047
|
+
return d;
|
|
71048
|
+
}
|
|
71049
|
+
title.textContent = 'Importing your data\u2026';
|
|
71050
|
+
addLine('Starting\u2026');
|
|
71051
|
+
|
|
71052
|
+
iiStreamNdjson('/api/import/apply', { fileId: fileId, mode: mode, asOf: asOf, asOfColumn: asOfColumn }, function (evt) {
|
|
71053
|
+
if (!evt) return;
|
|
71054
|
+
if (evt.phase === 'done') {
|
|
71055
|
+
var r = evt.result || {};
|
|
71056
|
+
var rbt = r.rowsByTable || {};
|
|
71057
|
+
var names = Object.keys(rbt);
|
|
71058
|
+
var total = 0;
|
|
71059
|
+
names.forEach(function (n) { total += (rbt[n] || 0); });
|
|
71060
|
+
title.textContent = 'Imported ' + names.length + ' tables' + (mode === 'schema' ? '' : ', ' + total + ' rows');
|
|
71061
|
+
var upd = addLine('Updating your objects\u2026', 'imp-spin');
|
|
71062
|
+
refreshEntities().then(function () {
|
|
71063
|
+
renderSidebar();
|
|
71064
|
+
renderRoute();
|
|
71065
|
+
var count = (state.entities && state.entities.tables) ? state.entities.tables.length : names.length;
|
|
71066
|
+
if (upd) {
|
|
71067
|
+
upd.className = 'imp-card-line imp-done';
|
|
71068
|
+
upd.textContent = '\u2713 Done \u2014 ' + count + ' objects in your workspace';
|
|
71069
|
+
}
|
|
71070
|
+
}).catch(function () {
|
|
71071
|
+
if (upd) {
|
|
71072
|
+
upd.className = 'imp-card-line imp-err';
|
|
71073
|
+
upd.textContent = 'Imported, but refreshing the view failed \u2014 reload to see your objects.';
|
|
71074
|
+
}
|
|
71075
|
+
});
|
|
71076
|
+
} else if (evt.phase === 'error') {
|
|
71077
|
+
title.textContent = 'Import failed';
|
|
71078
|
+
addLine('Error: ' + (evt.message || 'import failed'), 'imp-err');
|
|
71079
|
+
} else if (evt.message) {
|
|
71080
|
+
addLine(evt.message);
|
|
71081
|
+
}
|
|
71082
|
+
});
|
|
71083
|
+
}
|
|
71084
|
+
`;
|
|
71085
|
+
|
|
70547
71086
|
// src/gui/app/modules/index.ts
|
|
70548
71087
|
var appJs = [
|
|
70549
71088
|
displayConfigJs,
|
|
@@ -70572,7 +71111,8 @@ var appJs = [
|
|
|
70572
71111
|
dataModelJs,
|
|
70573
71112
|
latticeTeamsJs,
|
|
70574
71113
|
onboardingJs,
|
|
70575
|
-
createDatabaseWizardJs
|
|
71114
|
+
createDatabaseWizardJs,
|
|
71115
|
+
inlineImportJs
|
|
70576
71116
|
].join("");
|
|
70577
71117
|
|
|
70578
71118
|
// src/gui/app/analytics.ts
|
|
@@ -71431,9 +71971,9 @@ function rewriteDbLine(configPath, newValue) {
|
|
|
71431
71971
|
function parseSaveBody(body) {
|
|
71432
71972
|
const type = body.type;
|
|
71433
71973
|
if (type === "sqlite") {
|
|
71434
|
-
const
|
|
71435
|
-
if (!
|
|
71436
|
-
return { type: "sqlite", path:
|
|
71974
|
+
const path3 = typeof body.path === "string" && body.path.trim() ? body.path.trim() : "";
|
|
71975
|
+
if (!path3) return null;
|
|
71976
|
+
return { type: "sqlite", path: path3 };
|
|
71437
71977
|
}
|
|
71438
71978
|
if (type === "postgres") {
|
|
71439
71979
|
const label = typeof body.label === "string" && body.label.trim() ? body.label.trim() : "";
|
|
@@ -72952,14 +73492,1167 @@ init_extract();
|
|
|
72952
73492
|
import { statSync as statSync9 } from "fs";
|
|
72953
73493
|
import { writeFile as writeFile2, rm } from "fs/promises";
|
|
72954
73494
|
import { tmpdir as tmpdir2 } from "os";
|
|
72955
|
-
import { basename as
|
|
73495
|
+
import { basename as basename12, extname as extname2, resolve as resolve11, join as join29 } from "path";
|
|
72956
73496
|
init_assistant_routes();
|
|
72957
73497
|
init_http2();
|
|
72958
73498
|
init_enrich();
|
|
72959
73499
|
init_ingest_url();
|
|
72960
73500
|
init_file_row();
|
|
72961
|
-
import { createHash as
|
|
73501
|
+
import { createHash as createHash14 } from "crypto";
|
|
72962
73502
|
init_dedup_service();
|
|
73503
|
+
|
|
73504
|
+
// src/gui/import-auto.ts
|
|
73505
|
+
import { readFileSync as readFileSync23 } from "fs";
|
|
73506
|
+
|
|
73507
|
+
// src/import/infer.ts
|
|
73508
|
+
var SAMPLE = 300;
|
|
73509
|
+
var PREFERRED_KEYS = ["code", "id", "slug", "key", "ticker", "symbol"];
|
|
73510
|
+
var NEVER_KEY = /* @__PURE__ */ new Set([
|
|
73511
|
+
"description",
|
|
73512
|
+
"notes",
|
|
73513
|
+
"summary",
|
|
73514
|
+
"desc",
|
|
73515
|
+
"comment",
|
|
73516
|
+
"comments",
|
|
73517
|
+
"bio",
|
|
73518
|
+
"text",
|
|
73519
|
+
"body"
|
|
73520
|
+
]);
|
|
73521
|
+
var FREETEXT = /* @__PURE__ */ new Set([...NEVER_KEY, "name", "title", "company", "label"]);
|
|
73522
|
+
var DIM_MAX_DISTINCT = 64;
|
|
73523
|
+
var DIM_MAX_RATIO = 0.5;
|
|
73524
|
+
var LINK_MIN_CONFIDENCE = 0.3;
|
|
73525
|
+
function isPlainObject(v2) {
|
|
73526
|
+
return typeof v2 === "object" && v2 !== null && !Array.isArray(v2);
|
|
73527
|
+
}
|
|
73528
|
+
function sourceRecords(data, entity) {
|
|
73529
|
+
const v2 = data[entity.sourceKey];
|
|
73530
|
+
if (!Array.isArray(v2)) return [];
|
|
73531
|
+
if (entity.columnar) {
|
|
73532
|
+
const cols = data[entity.sourceKey + "Cols"];
|
|
73533
|
+
if (!Array.isArray(cols)) return [];
|
|
73534
|
+
return v2.map((row) => {
|
|
73535
|
+
const o3 = {};
|
|
73536
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
73537
|
+
return o3;
|
|
73538
|
+
});
|
|
73539
|
+
}
|
|
73540
|
+
return v2.filter(isPlainObject);
|
|
73541
|
+
}
|
|
73542
|
+
function normalizeName(key) {
|
|
73543
|
+
const s2 = key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
73544
|
+
if (!s2) return "field";
|
|
73545
|
+
return /^[a-z]/.test(s2) ? s2 : "f_" + s2;
|
|
73546
|
+
}
|
|
73547
|
+
var ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
73548
|
+
var ISO_DATETIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/;
|
|
73549
|
+
function inferFieldType(values) {
|
|
73550
|
+
const present = values.filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
73551
|
+
if (present.length === 0) return "text";
|
|
73552
|
+
if (present.every((v2) => typeof v2 === "number")) {
|
|
73553
|
+
return present.every((v2) => Number.isInteger(v2)) ? "integer" : "real";
|
|
73554
|
+
}
|
|
73555
|
+
if (present.every((v2) => typeof v2 === "boolean")) return "boolean";
|
|
73556
|
+
if (present.every((v2) => typeof v2 === "string")) {
|
|
73557
|
+
if (present.every((v2) => ISO_DATE.test(v2))) return "date";
|
|
73558
|
+
if (present.every((v2) => ISO_DATETIME.test(v2))) return "datetime";
|
|
73559
|
+
}
|
|
73560
|
+
return "text";
|
|
73561
|
+
}
|
|
73562
|
+
function norm2(v2) {
|
|
73563
|
+
return String(v2).trim().toLowerCase();
|
|
73564
|
+
}
|
|
73565
|
+
function isNumericValue(v2) {
|
|
73566
|
+
if (typeof v2 === "number") return Number.isFinite(v2);
|
|
73567
|
+
if (typeof v2 !== "string") return false;
|
|
73568
|
+
const s2 = v2.replace(/[\s,$%()]/g, "");
|
|
73569
|
+
return s2 !== "" && Number.isFinite(Number(s2));
|
|
73570
|
+
}
|
|
73571
|
+
function profileColumns(records) {
|
|
73572
|
+
const keys = /* @__PURE__ */ new Set();
|
|
73573
|
+
for (const r6 of records.slice(0, SAMPLE)) for (const k6 of Object.keys(r6)) keys.add(k6);
|
|
73574
|
+
const out = /* @__PURE__ */ new Map();
|
|
73575
|
+
for (const key of keys) {
|
|
73576
|
+
let isArray = false;
|
|
73577
|
+
const sample = [];
|
|
73578
|
+
const valueSet = /* @__PURE__ */ new Set();
|
|
73579
|
+
const distinctSet = /* @__PURE__ */ new Set();
|
|
73580
|
+
let nonNull = 0;
|
|
73581
|
+
let numeric = 0;
|
|
73582
|
+
for (const r6 of records) {
|
|
73583
|
+
const v2 = r6[key];
|
|
73584
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
73585
|
+
nonNull++;
|
|
73586
|
+
if (Array.isArray(v2)) {
|
|
73587
|
+
isArray = true;
|
|
73588
|
+
for (const e6 of v2) {
|
|
73589
|
+
if (e6 !== null && e6 !== void 0 && e6 !== "") {
|
|
73590
|
+
valueSet.add(norm2(e6));
|
|
73591
|
+
distinctSet.add(norm2(e6));
|
|
73592
|
+
}
|
|
73593
|
+
}
|
|
73594
|
+
} else {
|
|
73595
|
+
if (sample.length < SAMPLE) sample.push(v2);
|
|
73596
|
+
if (typeof v2 === "string") valueSet.add(norm2(v2));
|
|
73597
|
+
distinctSet.add(norm2(v2));
|
|
73598
|
+
if (isNumericValue(v2)) numeric++;
|
|
73599
|
+
}
|
|
73600
|
+
}
|
|
73601
|
+
out.set(key, {
|
|
73602
|
+
sourceKey: key,
|
|
73603
|
+
isArray,
|
|
73604
|
+
type: isArray ? "text" : inferFieldType(sample),
|
|
73605
|
+
// Cardinality counts ALL distinct values (numbers + strings). Counting only
|
|
73606
|
+
// string values let a mostly-numeric column with a few text sentinels (e.g.
|
|
73607
|
+
// a "TEV/EBITDA" of numbers + "NM") look low-cardinality and slip in as a
|
|
73608
|
+
// junk dimension.
|
|
73609
|
+
distinct: distinctSet.size,
|
|
73610
|
+
valueSet,
|
|
73611
|
+
numericFraction: nonNull > 0 ? numeric / nonNull : 0
|
|
73612
|
+
});
|
|
73613
|
+
}
|
|
73614
|
+
return out;
|
|
73615
|
+
}
|
|
73616
|
+
function pickNaturalKey(records, profiles) {
|
|
73617
|
+
const n3 = records.length;
|
|
73618
|
+
const isUnique = (key) => {
|
|
73619
|
+
const seen = /* @__PURE__ */ new Set();
|
|
73620
|
+
for (const r6 of records) {
|
|
73621
|
+
const v2 = r6[key];
|
|
73622
|
+
if (v2 === null || v2 === void 0 || v2 === "") return false;
|
|
73623
|
+
const k6 = norm2(v2);
|
|
73624
|
+
if (seen.has(k6)) return false;
|
|
73625
|
+
seen.add(k6);
|
|
73626
|
+
}
|
|
73627
|
+
return seen.size === n3;
|
|
73628
|
+
};
|
|
73629
|
+
for (const pref of PREFERRED_KEYS) {
|
|
73630
|
+
for (const [key, p3] of profiles) {
|
|
73631
|
+
if (p3.isArray) continue;
|
|
73632
|
+
if (normalizeName(key) === pref && isUnique(key)) return key;
|
|
73633
|
+
}
|
|
73634
|
+
}
|
|
73635
|
+
for (const [key, p3] of profiles) {
|
|
73636
|
+
if (p3.isArray) continue;
|
|
73637
|
+
if (NEVER_KEY.has(normalizeName(key))) continue;
|
|
73638
|
+
if ((p3.type === "text" || p3.type === "integer") && isUnique(key)) return key;
|
|
73639
|
+
}
|
|
73640
|
+
return null;
|
|
73641
|
+
}
|
|
73642
|
+
function inferSchema(data, opts = {}) {
|
|
73643
|
+
const skipped = [];
|
|
73644
|
+
const consumedColsKeys = /* @__PURE__ */ new Set();
|
|
73645
|
+
for (const key of Object.keys(data)) {
|
|
73646
|
+
const v2 = data[key];
|
|
73647
|
+
const cols = data[key + "Cols"];
|
|
73648
|
+
if (Array.isArray(v2) && v2.length > 0 && Array.isArray(v2[0]) && Array.isArray(cols) && cols.every((c6) => typeof c6 === "string")) {
|
|
73649
|
+
consumedColsKeys.add(key + "Cols");
|
|
73650
|
+
}
|
|
73651
|
+
}
|
|
73652
|
+
const sources = [];
|
|
73653
|
+
for (const key of Object.keys(data)) {
|
|
73654
|
+
if (consumedColsKeys.has(key)) continue;
|
|
73655
|
+
const v2 = data[key];
|
|
73656
|
+
if (!Array.isArray(v2) || v2.length === 0) {
|
|
73657
|
+
skipped.push({
|
|
73658
|
+
key,
|
|
73659
|
+
reason: isPlainObject(v2) ? "object (derived/rollup)" : "scalar/empty (meta or derived)"
|
|
73660
|
+
});
|
|
73661
|
+
continue;
|
|
73662
|
+
}
|
|
73663
|
+
let records;
|
|
73664
|
+
let columnar = false;
|
|
73665
|
+
if (isPlainObject(v2[0])) {
|
|
73666
|
+
records = v2.filter(isPlainObject);
|
|
73667
|
+
} else if (Array.isArray(v2[0]) && Array.isArray(data[key + "Cols"])) {
|
|
73668
|
+
const cols = data[key + "Cols"];
|
|
73669
|
+
records = v2.map((row) => {
|
|
73670
|
+
const o3 = {};
|
|
73671
|
+
cols.forEach((c6, i6) => o3[c6] = row[i6]);
|
|
73672
|
+
return o3;
|
|
73673
|
+
});
|
|
73674
|
+
columnar = true;
|
|
73675
|
+
} else {
|
|
73676
|
+
skipped.push({ key, reason: "array of scalars (not a record set)" });
|
|
73677
|
+
continue;
|
|
73678
|
+
}
|
|
73679
|
+
const name = opts.rename?.[key] ?? normalizeName(key);
|
|
73680
|
+
const profiles = profileColumns(records);
|
|
73681
|
+
sources.push({
|
|
73682
|
+
name,
|
|
73683
|
+
sourceKey: key,
|
|
73684
|
+
records,
|
|
73685
|
+
columnar,
|
|
73686
|
+
profiles,
|
|
73687
|
+
naturalKey: pickNaturalKey(records, profiles)
|
|
73688
|
+
});
|
|
73689
|
+
}
|
|
73690
|
+
const linkages = [];
|
|
73691
|
+
const consumedFields = /* @__PURE__ */ new Map();
|
|
73692
|
+
const linkedTargets = /* @__PURE__ */ new Map();
|
|
73693
|
+
const consume = (e6, f6) => {
|
|
73694
|
+
let set = consumedFields.get(e6);
|
|
73695
|
+
if (!set) {
|
|
73696
|
+
set = /* @__PURE__ */ new Set();
|
|
73697
|
+
consumedFields.set(e6, set);
|
|
73698
|
+
}
|
|
73699
|
+
set.add(f6);
|
|
73700
|
+
};
|
|
73701
|
+
const markTarget = (e6, t8) => {
|
|
73702
|
+
let set = linkedTargets.get(e6);
|
|
73703
|
+
if (!set) {
|
|
73704
|
+
set = /* @__PURE__ */ new Set();
|
|
73705
|
+
linkedTargets.set(e6, set);
|
|
73706
|
+
}
|
|
73707
|
+
set.add(t8);
|
|
73708
|
+
};
|
|
73709
|
+
function bestTarget(self, values) {
|
|
73710
|
+
if (values.size === 0) return null;
|
|
73711
|
+
let best = null;
|
|
73712
|
+
for (const t8 of sources) {
|
|
73713
|
+
if (t8.name === self.name || !t8.naturalKey) continue;
|
|
73714
|
+
const p3 = t8.profiles.get(t8.naturalKey);
|
|
73715
|
+
if (!p3 || p3.valueSet.size === 0) continue;
|
|
73716
|
+
let matched = 0;
|
|
73717
|
+
for (const v2 of values) if (p3.valueSet.has(v2)) matched++;
|
|
73718
|
+
if (matched > 0 && (best === null || matched > best.matched)) {
|
|
73719
|
+
best = { target: t8, column: t8.naturalKey, matched };
|
|
73720
|
+
}
|
|
73721
|
+
}
|
|
73722
|
+
return best;
|
|
73723
|
+
}
|
|
73724
|
+
for (const pass of ["array", "scalar"]) {
|
|
73725
|
+
for (const e6 of sources) {
|
|
73726
|
+
for (const [field, p3] of e6.profiles) {
|
|
73727
|
+
if (pass === "array" ? !p3.isArray : p3.isArray) continue;
|
|
73728
|
+
if (pass === "scalar") {
|
|
73729
|
+
if (field === e6.naturalKey) continue;
|
|
73730
|
+
if (FREETEXT.has(normalizeName(field)) || NEVER_KEY.has(normalizeName(field))) continue;
|
|
73731
|
+
if (p3.type !== "text") continue;
|
|
73732
|
+
}
|
|
73733
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
73734
|
+
const best = bestTarget(e6, p3.valueSet);
|
|
73735
|
+
if (!best) continue;
|
|
73736
|
+
const confidence = best.matched / p3.valueSet.size;
|
|
73737
|
+
if (confidence < LINK_MIN_CONFIDENCE) continue;
|
|
73738
|
+
if (linkedTargets.get(e6.name)?.has(best.target.name)) {
|
|
73739
|
+
consume(e6.name, field);
|
|
73740
|
+
continue;
|
|
73741
|
+
}
|
|
73742
|
+
const link = {
|
|
73743
|
+
kind: pass === "array" ? "many-to-many" : "many-to-one",
|
|
73744
|
+
fromEntity: e6.name,
|
|
73745
|
+
fromField: field,
|
|
73746
|
+
toEntity: best.target.name,
|
|
73747
|
+
toKey: normalizeName(best.column),
|
|
73748
|
+
matched: best.matched,
|
|
73749
|
+
unresolved: p3.valueSet.size - best.matched,
|
|
73750
|
+
confidence
|
|
73751
|
+
};
|
|
73752
|
+
if (pass === "array") link.junction = `${e6.name}_${best.target.name}`;
|
|
73753
|
+
linkages.push(link);
|
|
73754
|
+
consume(e6.name, field);
|
|
73755
|
+
markTarget(e6.name, best.target.name);
|
|
73756
|
+
}
|
|
73757
|
+
}
|
|
73758
|
+
}
|
|
73759
|
+
const dimColumnNames = /* @__PURE__ */ new Map();
|
|
73760
|
+
for (const e6 of sources) {
|
|
73761
|
+
for (const [field, p3] of e6.profiles) {
|
|
73762
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
73763
|
+
const nn = normalizeName(field);
|
|
73764
|
+
let arr = dimColumnNames.get(nn);
|
|
73765
|
+
if (!arr) {
|
|
73766
|
+
arr = [];
|
|
73767
|
+
dimColumnNames.set(nn, arr);
|
|
73768
|
+
}
|
|
73769
|
+
arr.push(e6);
|
|
73770
|
+
}
|
|
73771
|
+
}
|
|
73772
|
+
const dimensions = [];
|
|
73773
|
+
const dimByName = /* @__PURE__ */ new Map();
|
|
73774
|
+
for (const e6 of sources) {
|
|
73775
|
+
for (const [field, p3] of e6.profiles) {
|
|
73776
|
+
if (p3.isArray || p3.type !== "text" || p3.numericFraction > 0.5) continue;
|
|
73777
|
+
if (field === e6.naturalKey) continue;
|
|
73778
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
73779
|
+
const nn = normalizeName(field);
|
|
73780
|
+
if (FREETEXT.has(nn)) continue;
|
|
73781
|
+
const ratio = p3.distinct / Math.max(1, e6.records.length);
|
|
73782
|
+
const sharedAcross = dimColumnNames.get(nn)?.length ?? 1;
|
|
73783
|
+
const isDim = p3.distinct >= 1 && p3.distinct <= DIM_MAX_DISTINCT && (ratio <= DIM_MAX_RATIO || sharedAcross >= 2);
|
|
73784
|
+
if (!isDim) continue;
|
|
73785
|
+
let dim = dimByName.get(nn);
|
|
73786
|
+
if (!dim) {
|
|
73787
|
+
dim = { name: nn, sourceField: field, fromEntities: [], distinctValues: 0 };
|
|
73788
|
+
dimByName.set(nn, dim);
|
|
73789
|
+
dimensions.push(dim);
|
|
73790
|
+
}
|
|
73791
|
+
if (!dim.fromEntities.includes(e6.name)) dim.fromEntities.push(e6.name);
|
|
73792
|
+
linkages.push({
|
|
73793
|
+
kind: "dimension",
|
|
73794
|
+
fromEntity: e6.name,
|
|
73795
|
+
fromField: field,
|
|
73796
|
+
toEntity: nn,
|
|
73797
|
+
toKey: "value",
|
|
73798
|
+
junction: `${e6.name}_${nn}`,
|
|
73799
|
+
matched: p3.distinct,
|
|
73800
|
+
unresolved: 0,
|
|
73801
|
+
confidence: 1
|
|
73802
|
+
});
|
|
73803
|
+
consume(e6.name, field);
|
|
73804
|
+
}
|
|
73805
|
+
}
|
|
73806
|
+
for (const dim of dimensions) {
|
|
73807
|
+
const all = /* @__PURE__ */ new Set();
|
|
73808
|
+
for (const name of dim.fromEntities) {
|
|
73809
|
+
const e6 = sources.find((s2) => s2.name === name);
|
|
73810
|
+
if (!e6) continue;
|
|
73811
|
+
for (const [f6, p3] of e6.profiles) {
|
|
73812
|
+
if (normalizeName(f6) === dim.name) for (const v2 of p3.valueSet) all.add(v2);
|
|
73813
|
+
}
|
|
73814
|
+
}
|
|
73815
|
+
dim.distinctValues = all.size;
|
|
73816
|
+
}
|
|
73817
|
+
const entities = sources.map((e6) => {
|
|
73818
|
+
const columns = [];
|
|
73819
|
+
for (const [field, p3] of e6.profiles) {
|
|
73820
|
+
if (p3.isArray) continue;
|
|
73821
|
+
if (consumedFields.get(e6.name)?.has(field)) continue;
|
|
73822
|
+
columns.push({ name: normalizeName(field), sourceKey: field, type: p3.type });
|
|
73823
|
+
}
|
|
73824
|
+
return {
|
|
73825
|
+
name: e6.name,
|
|
73826
|
+
sourceKey: e6.sourceKey,
|
|
73827
|
+
columns,
|
|
73828
|
+
naturalKey: e6.naturalKey ? normalizeName(e6.naturalKey) : null,
|
|
73829
|
+
naturalKeySource: e6.naturalKey,
|
|
73830
|
+
rowCount: e6.records.length,
|
|
73831
|
+
columnar: e6.columnar
|
|
73832
|
+
};
|
|
73833
|
+
});
|
|
73834
|
+
return { entities, dimensions, linkages, skipped };
|
|
73835
|
+
}
|
|
73836
|
+
|
|
73837
|
+
// src/import/dedupe-views.ts
|
|
73838
|
+
init_normalize();
|
|
73839
|
+
var SAMPLE2 = 300;
|
|
73840
|
+
var VIEW_MIN_OVERLAP = 0.8;
|
|
73841
|
+
function buildEntityData(plan, data) {
|
|
73842
|
+
return plan.entities.map((e6) => {
|
|
73843
|
+
const records = sourceRecords(data, e6);
|
|
73844
|
+
const colSet = /* @__PURE__ */ new Set();
|
|
73845
|
+
const colSource = /* @__PURE__ */ new Map();
|
|
73846
|
+
for (const r6 of records.slice(0, SAMPLE2)) {
|
|
73847
|
+
for (const k6 of Object.keys(r6)) {
|
|
73848
|
+
const n3 = normalizeName(k6);
|
|
73849
|
+
colSet.add(n3);
|
|
73850
|
+
if (!colSource.has(n3)) colSource.set(n3, k6);
|
|
73851
|
+
}
|
|
73852
|
+
}
|
|
73853
|
+
const normRows = records.map((r6) => {
|
|
73854
|
+
const o3 = {};
|
|
73855
|
+
for (const k6 of Object.keys(r6)) o3[normalizeName(k6)] = r6[k6];
|
|
73856
|
+
return o3;
|
|
73857
|
+
});
|
|
73858
|
+
return { name: e6.name, sourceKey: e6.sourceKey, cols: [...colSet], colSource, normRows };
|
|
73859
|
+
});
|
|
73860
|
+
}
|
|
73861
|
+
function pickIdentity(a6, shared) {
|
|
73862
|
+
let bestCol = null;
|
|
73863
|
+
let bestDistinct = -1;
|
|
73864
|
+
for (const c6 of shared) {
|
|
73865
|
+
const vals = /* @__PURE__ */ new Set();
|
|
73866
|
+
let textish = 0;
|
|
73867
|
+
let total = 0;
|
|
73868
|
+
for (const r6 of a6.normRows) {
|
|
73869
|
+
const v2 = r6[c6];
|
|
73870
|
+
if (v2 === null || v2 === void 0 || v2 === "") continue;
|
|
73871
|
+
total++;
|
|
73872
|
+
if (typeof v2 === "string") textish++;
|
|
73873
|
+
vals.add(normalizeText(v2));
|
|
73874
|
+
}
|
|
73875
|
+
if (total === 0 || textish / total < 0.7) continue;
|
|
73876
|
+
if (vals.size > bestDistinct) {
|
|
73877
|
+
bestDistinct = vals.size;
|
|
73878
|
+
bestCol = c6;
|
|
73879
|
+
}
|
|
73880
|
+
}
|
|
73881
|
+
return bestCol;
|
|
73882
|
+
}
|
|
73883
|
+
function dedupeAndDetectViews(plan, data) {
|
|
73884
|
+
const entities = buildEntityData(plan, data);
|
|
73885
|
+
const views = [];
|
|
73886
|
+
const asView = /* @__PURE__ */ new Set();
|
|
73887
|
+
const colKeeps = [];
|
|
73888
|
+
for (const a6 of entities) {
|
|
73889
|
+
if (a6.cols.length < 2 || a6.normRows.length === 0) continue;
|
|
73890
|
+
const tabName = normalizeText(a6.sourceKey);
|
|
73891
|
+
if (!tabName) continue;
|
|
73892
|
+
const aColSet = new Set(a6.cols);
|
|
73893
|
+
let best = null;
|
|
73894
|
+
for (const b6 of entities) {
|
|
73895
|
+
if (b6.name === a6.name || asView.has(b6.name)) continue;
|
|
73896
|
+
if (b6.normRows.length < a6.normRows.length) continue;
|
|
73897
|
+
const bColSet = new Set(b6.cols);
|
|
73898
|
+
const shared = a6.cols.filter((c6) => bColSet.has(c6));
|
|
73899
|
+
if (shared.length < Math.max(2, Math.ceil(a6.cols.length * 0.5))) continue;
|
|
73900
|
+
const identity = pickIdentity(a6, shared);
|
|
73901
|
+
if (!identity) continue;
|
|
73902
|
+
const aIds = new Set(
|
|
73903
|
+
a6.normRows.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== "")
|
|
73904
|
+
);
|
|
73905
|
+
if (aIds.size === 0) continue;
|
|
73906
|
+
for (const disc of b6.cols) {
|
|
73907
|
+
if (aColSet.has(disc)) continue;
|
|
73908
|
+
const sub = b6.normRows.filter((r6) => normalizeText(r6[disc]) === tabName);
|
|
73909
|
+
if (sub.length === 0) continue;
|
|
73910
|
+
const bIds = new Set(sub.map((r6) => normalizeText(r6[identity])).filter((v2) => v2 !== ""));
|
|
73911
|
+
let inter = 0;
|
|
73912
|
+
for (const id of aIds) if (bIds.has(id)) inter++;
|
|
73913
|
+
const overlap = inter / aIds.size;
|
|
73914
|
+
if (overlap < VIEW_MIN_OVERLAP) continue;
|
|
73915
|
+
const rawRow = sub.find((r6) => typeof r6[disc] === "string" || typeof r6[disc] === "number");
|
|
73916
|
+
const raw = rawRow ? rawRow[disc] : void 0;
|
|
73917
|
+
if (typeof raw !== "string" && typeof raw !== "number") continue;
|
|
73918
|
+
if (best === null || overlap > best.overlap || overlap === best.overlap && b6.cols.length > best.master.cols.length) {
|
|
73919
|
+
best = { master: b6, disc, value: String(raw), matched: sub.length, overlap };
|
|
73920
|
+
}
|
|
73921
|
+
}
|
|
73922
|
+
}
|
|
73923
|
+
if (!best) continue;
|
|
73924
|
+
views.push({
|
|
73925
|
+
name: a6.name,
|
|
73926
|
+
master: best.master.name,
|
|
73927
|
+
filterColumn: best.disc,
|
|
73928
|
+
filterValue: best.value,
|
|
73929
|
+
matchedRows: best.matched
|
|
73930
|
+
});
|
|
73931
|
+
asView.add(a6.name);
|
|
73932
|
+
colKeeps.push({ master: best.master, col: best.disc });
|
|
73933
|
+
}
|
|
73934
|
+
for (const { master, col } of colKeeps) {
|
|
73935
|
+
const masterEntity = plan.entities.find((e6) => e6.name === master.name);
|
|
73936
|
+
if (!masterEntity || masterEntity.columns.some((c6) => c6.name === col)) continue;
|
|
73937
|
+
masterEntity.columns.push({
|
|
73938
|
+
name: col,
|
|
73939
|
+
sourceKey: master.colSource.get(col) ?? col,
|
|
73940
|
+
type: inferFieldType(master.normRows.map((r6) => r6[col]))
|
|
73941
|
+
});
|
|
73942
|
+
}
|
|
73943
|
+
if (views.length === 0) return { plan, views };
|
|
73944
|
+
const nextPlan = {
|
|
73945
|
+
entities: plan.entities.filter((e6) => !asView.has(e6.name)),
|
|
73946
|
+
linkages: plan.linkages.filter((l4) => !asView.has(l4.fromEntity)),
|
|
73947
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.filter((n3) => !asView.has(n3)) })).filter((d6) => d6.fromEntities.length > 0),
|
|
73948
|
+
skipped: plan.skipped
|
|
73949
|
+
};
|
|
73950
|
+
return { plan: nextPlan, views };
|
|
73951
|
+
}
|
|
73952
|
+
|
|
73953
|
+
// src/import/excel.ts
|
|
73954
|
+
import { resolve as resolve10 } from "path";
|
|
73955
|
+
var HEADER_SCAN_ROWS = 25;
|
|
73956
|
+
function cellValue(v2) {
|
|
73957
|
+
if (v2 === null || v2 === void 0) return null;
|
|
73958
|
+
if (v2 instanceof Date) return v2.toISOString().slice(0, 10);
|
|
73959
|
+
if (typeof v2 === "object") {
|
|
73960
|
+
const o3 = v2;
|
|
73961
|
+
if ("result" in o3) return cellValue(o3.result);
|
|
73962
|
+
if ("text" in o3) return o3.text;
|
|
73963
|
+
if ("richText" in o3 && Array.isArray(o3.richText)) {
|
|
73964
|
+
return o3.richText.map((t8) => t8.text ?? "").join("");
|
|
73965
|
+
}
|
|
73966
|
+
return null;
|
|
73967
|
+
}
|
|
73968
|
+
return v2;
|
|
73969
|
+
}
|
|
73970
|
+
function isFilled(v2) {
|
|
73971
|
+
return v2 !== null && v2 !== void 0 && v2 !== "";
|
|
73972
|
+
}
|
|
73973
|
+
function sheetToRecords(ws) {
|
|
73974
|
+
const rowCount = ws.rowCount;
|
|
73975
|
+
const colCount = ws.columnCount;
|
|
73976
|
+
if (rowCount < 2 || colCount < 2) return [];
|
|
73977
|
+
const nonEmpty = (r6) => {
|
|
73978
|
+
let n3 = 0;
|
|
73979
|
+
for (let c6 = 1; c6 <= colCount; c6++) if (isFilled(cellValue(ws.getCell(r6, c6).value))) n3++;
|
|
73980
|
+
return n3;
|
|
73981
|
+
};
|
|
73982
|
+
const threshold = Math.max(3, Math.floor(colCount * 0.4));
|
|
73983
|
+
let headerRow = -1;
|
|
73984
|
+
for (let r6 = 1; r6 <= Math.min(HEADER_SCAN_ROWS, rowCount); r6++) {
|
|
73985
|
+
if (nonEmpty(r6) >= threshold && r6 < rowCount && nonEmpty(r6 + 1) >= 2) {
|
|
73986
|
+
headerRow = r6;
|
|
73987
|
+
break;
|
|
73988
|
+
}
|
|
73989
|
+
}
|
|
73990
|
+
if (headerRow < 0) return [];
|
|
73991
|
+
const cols = [];
|
|
73992
|
+
const seen = /* @__PURE__ */ new Set();
|
|
73993
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
73994
|
+
const hv = cellValue(ws.getCell(headerRow, c6).value);
|
|
73995
|
+
if (!isFilled(hv)) continue;
|
|
73996
|
+
const base = String(hv).replace(/\s+/g, " ").trim();
|
|
73997
|
+
if (!base) continue;
|
|
73998
|
+
let name = base;
|
|
73999
|
+
let i6 = 2;
|
|
74000
|
+
while (seen.has(name)) name = base + " " + String(i6++);
|
|
74001
|
+
seen.add(name);
|
|
74002
|
+
cols.push({ c: c6, name });
|
|
74003
|
+
}
|
|
74004
|
+
if (cols.length === 0) return [];
|
|
74005
|
+
const records = [];
|
|
74006
|
+
for (let r6 = headerRow + 1; r6 <= rowCount; r6++) {
|
|
74007
|
+
const row = {};
|
|
74008
|
+
let any = false;
|
|
74009
|
+
for (const { c: c6, name } of cols) {
|
|
74010
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
74011
|
+
if (isFilled(v2)) {
|
|
74012
|
+
row[name] = v2;
|
|
74013
|
+
any = true;
|
|
74014
|
+
}
|
|
74015
|
+
}
|
|
74016
|
+
if (!any) break;
|
|
74017
|
+
const first = cols[0] ? row[cols[0].name] : void 0;
|
|
74018
|
+
if (typeof first === "string" && /^total\b/i.test(first.trim())) continue;
|
|
74019
|
+
records.push(row);
|
|
74020
|
+
}
|
|
74021
|
+
return records;
|
|
74022
|
+
}
|
|
74023
|
+
var preambleCache = /* @__PURE__ */ new Map();
|
|
74024
|
+
function excelPreambleText(absPath) {
|
|
74025
|
+
return preambleCache.get(resolve10(absPath)) ?? "";
|
|
74026
|
+
}
|
|
74027
|
+
function sheetPreamble(ws) {
|
|
74028
|
+
const lines = [];
|
|
74029
|
+
const rowCount = Math.min(10, ws.rowCount);
|
|
74030
|
+
const colCount = Math.min(8, ws.columnCount);
|
|
74031
|
+
for (let r6 = 1; r6 <= rowCount; r6++) {
|
|
74032
|
+
const cells = [];
|
|
74033
|
+
for (let c6 = 1; c6 <= colCount; c6++) {
|
|
74034
|
+
const v2 = cellValue(ws.getCell(r6, c6).value);
|
|
74035
|
+
if (isFilled(v2)) cells.push(String(v2));
|
|
74036
|
+
}
|
|
74037
|
+
if (cells.length) lines.push(cells.join(" "));
|
|
74038
|
+
}
|
|
74039
|
+
return lines.join("\n");
|
|
74040
|
+
}
|
|
74041
|
+
async function excelToRecords(absPath) {
|
|
74042
|
+
let mod;
|
|
74043
|
+
try {
|
|
74044
|
+
mod = await import("exceljs");
|
|
74045
|
+
} catch {
|
|
74046
|
+
throw new Error(
|
|
74047
|
+
'Reading Excel files needs the "exceljs" package \u2014 install it with: npm install exceljs'
|
|
74048
|
+
);
|
|
74049
|
+
}
|
|
74050
|
+
const ExcelJS = mod.default ?? mod;
|
|
74051
|
+
const wb = new ExcelJS.Workbook();
|
|
74052
|
+
await wb.xlsx.readFile(absPath);
|
|
74053
|
+
const out = {};
|
|
74054
|
+
const preamble = [];
|
|
74055
|
+
const props = wb.properties;
|
|
74056
|
+
if (props?.title) preamble.push(props.title);
|
|
74057
|
+
for (const ws of wb.worksheets) {
|
|
74058
|
+
preamble.push(ws.name, sheetPreamble(ws));
|
|
74059
|
+
const records = sheetToRecords(ws);
|
|
74060
|
+
if (records.length > 0) out[ws.name] = records;
|
|
74061
|
+
}
|
|
74062
|
+
preambleCache.set(resolve10(absPath), preamble.filter(Boolean).join("\n"));
|
|
74063
|
+
return out;
|
|
74064
|
+
}
|
|
74065
|
+
|
|
74066
|
+
// src/import/match.ts
|
|
74067
|
+
var BOOKKEEPING = /* @__PURE__ */ new Set(["id", "as_of", "content_key", "deleted_at"]);
|
|
74068
|
+
var MATCH_THRESHOLD = 0.6;
|
|
74069
|
+
function signature(columns) {
|
|
74070
|
+
const out = /* @__PURE__ */ new Set();
|
|
74071
|
+
for (const c6 of columns) {
|
|
74072
|
+
const n3 = normalizeName(c6);
|
|
74073
|
+
if (!n3 || BOOKKEEPING.has(n3) || n3.endsWith("_id")) continue;
|
|
74074
|
+
out.add(n3);
|
|
74075
|
+
}
|
|
74076
|
+
return out;
|
|
74077
|
+
}
|
|
74078
|
+
function containment(a6, b6) {
|
|
74079
|
+
if (a6.size === 0) return 0;
|
|
74080
|
+
let hit = 0;
|
|
74081
|
+
for (const c6 of a6) if (b6.has(c6)) hit++;
|
|
74082
|
+
return hit / a6.size;
|
|
74083
|
+
}
|
|
74084
|
+
function matchSchemaToExisting(existing, plan) {
|
|
74085
|
+
const ex = existing.map((t8) => ({ name: t8.name, sig: signature(t8.columns) }));
|
|
74086
|
+
const matches = [];
|
|
74087
|
+
const rename = {};
|
|
74088
|
+
for (const ent of plan.entities) {
|
|
74089
|
+
const sig = signature(ent.columns.map((c6) => c6.name));
|
|
74090
|
+
if (sig.size === 0) continue;
|
|
74091
|
+
let best = null;
|
|
74092
|
+
for (const t8 of ex) {
|
|
74093
|
+
if (normalizeName(t8.name) === normalizeName(ent.name)) {
|
|
74094
|
+
best = { name: t8.name, overlap: 1 };
|
|
74095
|
+
break;
|
|
74096
|
+
}
|
|
74097
|
+
const overlap = containment(sig, t8.sig);
|
|
74098
|
+
if (overlap > (best?.overlap ?? 0)) best = { name: t8.name, overlap };
|
|
74099
|
+
}
|
|
74100
|
+
if (best && best.overlap >= MATCH_THRESHOLD) {
|
|
74101
|
+
matches.push({ from: ent.name, to: best.name, overlap: best.overlap });
|
|
74102
|
+
if (best.name !== ent.name) rename[ent.name] = best.name;
|
|
74103
|
+
}
|
|
74104
|
+
}
|
|
74105
|
+
const totalEntities = plan.entities.length;
|
|
74106
|
+
const matchedCount = matches.length;
|
|
74107
|
+
const isKnownDocument = totalEntities > 0 && matchedCount >= Math.ceil(totalEntities / 2);
|
|
74108
|
+
return { matches, rename, matchedCount, totalEntities, isKnownDocument };
|
|
74109
|
+
}
|
|
74110
|
+
function renameEntities(plan, views, rename) {
|
|
74111
|
+
if (Object.keys(rename).length === 0) return { plan, views };
|
|
74112
|
+
const r6 = (n3) => rename[n3] ?? n3;
|
|
74113
|
+
return {
|
|
74114
|
+
plan: {
|
|
74115
|
+
...plan,
|
|
74116
|
+
entities: plan.entities.map((e6) => ({ ...e6, name: r6(e6.name) })),
|
|
74117
|
+
dimensions: plan.dimensions.map((d6) => ({ ...d6, fromEntities: d6.fromEntities.map(r6) })),
|
|
74118
|
+
linkages: plan.linkages.map((l4) => ({
|
|
74119
|
+
...l4,
|
|
74120
|
+
fromEntity: r6(l4.fromEntity),
|
|
74121
|
+
toEntity: r6(l4.toEntity),
|
|
74122
|
+
...l4.junction ? { junction: l4.junction } : {}
|
|
74123
|
+
}))
|
|
74124
|
+
},
|
|
74125
|
+
views: views.map((v2) => ({ ...v2, name: r6(v2.name), master: r6(v2.master) }))
|
|
74126
|
+
};
|
|
74127
|
+
}
|
|
74128
|
+
|
|
74129
|
+
// src/import/materialize.ts
|
|
74130
|
+
init_parser();
|
|
74131
|
+
import { createHash as createHash13 } from "crypto";
|
|
74132
|
+
import { existsSync as existsSync25 } from "fs";
|
|
74133
|
+
init_normalize();
|
|
74134
|
+
|
|
74135
|
+
// src/import/asof.ts
|
|
74136
|
+
var MONTHS2 = {
|
|
74137
|
+
jan: 1,
|
|
74138
|
+
january: 1,
|
|
74139
|
+
feb: 2,
|
|
74140
|
+
february: 2,
|
|
74141
|
+
mar: 3,
|
|
74142
|
+
march: 3,
|
|
74143
|
+
apr: 4,
|
|
74144
|
+
april: 4,
|
|
74145
|
+
may: 5,
|
|
74146
|
+
jun: 6,
|
|
74147
|
+
june: 6,
|
|
74148
|
+
jul: 7,
|
|
74149
|
+
july: 7,
|
|
74150
|
+
aug: 8,
|
|
74151
|
+
august: 8,
|
|
74152
|
+
sep: 9,
|
|
74153
|
+
sept: 9,
|
|
74154
|
+
september: 9,
|
|
74155
|
+
oct: 10,
|
|
74156
|
+
october: 10,
|
|
74157
|
+
nov: 11,
|
|
74158
|
+
november: 11,
|
|
74159
|
+
dec: 12,
|
|
74160
|
+
december: 12
|
|
74161
|
+
};
|
|
74162
|
+
var ASOF_KEYWORDS = /\b(as[ -]?of|as at|period (?:end(?:ed|ing)?|of)|fye|fiscal year end(?:ed|ing)?|year[ -]?end(?:ed|ing)?|quarter[ -]?end(?:ed|ing)?|valuation date|report(?:ing)? date|effective date|dated)\b/i;
|
|
74163
|
+
function isoFrom(y2, m4, d6) {
|
|
74164
|
+
if (m4 < 1 || m4 > 12 || d6 < 1 || d6 > 31) return null;
|
|
74165
|
+
if (y2 < 2010 || y2 > 2099) return null;
|
|
74166
|
+
return `${String(y2)}-${String(m4).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
74167
|
+
}
|
|
74168
|
+
function findDates(text) {
|
|
74169
|
+
const hits = [];
|
|
74170
|
+
const push = (date2, match, index) => {
|
|
74171
|
+
if (date2) hits.push({ date: date2, match, index });
|
|
74172
|
+
};
|
|
74173
|
+
for (const m4 of text.matchAll(/(20\d{2})[-._/](\d{1,2})[-._/](\d{1,2})/g)) {
|
|
74174
|
+
push(isoFrom(Number(m4[1]), Number(m4[2]), Number(m4[3])), m4[0], m4.index);
|
|
74175
|
+
}
|
|
74176
|
+
for (const m4 of text.matchAll(/(\d{1,2})[-._/](\d{1,2})[-._/](\d{2,4})/g)) {
|
|
74177
|
+
let y2 = Number(m4[3]);
|
|
74178
|
+
if (y2 < 100) y2 += 2e3;
|
|
74179
|
+
push(isoFrom(y2, Number(m4[1]), Number(m4[2])), m4[0], m4.index);
|
|
74180
|
+
}
|
|
74181
|
+
for (const m4 of text.matchAll(/([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(20\d{2})/g)) {
|
|
74182
|
+
const mon = MONTHS2[(m4[1] ?? "").toLowerCase()];
|
|
74183
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[2])), m4[0], m4.index);
|
|
74184
|
+
}
|
|
74185
|
+
for (const m4 of text.matchAll(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(20\d{2})/g)) {
|
|
74186
|
+
const mon = MONTHS2[(m4[2] ?? "").toLowerCase()];
|
|
74187
|
+
if (mon) push(isoFrom(Number(m4[3]), mon, Number(m4[1])), m4[0], m4.index);
|
|
74188
|
+
}
|
|
74189
|
+
return hits;
|
|
74190
|
+
}
|
|
74191
|
+
function parseCellDate(value) {
|
|
74192
|
+
if (value instanceof Date) {
|
|
74193
|
+
return isoFrom(value.getUTCFullYear(), value.getUTCMonth() + 1, value.getUTCDate());
|
|
74194
|
+
}
|
|
74195
|
+
if (typeof value === "string") return findDates(value)[0]?.date ?? null;
|
|
74196
|
+
return null;
|
|
74197
|
+
}
|
|
74198
|
+
function scanText(text, label) {
|
|
74199
|
+
if (!text) return [];
|
|
74200
|
+
const out = [];
|
|
74201
|
+
for (const hit of findDates(text)) {
|
|
74202
|
+
const before = text.slice(Math.max(0, hit.index - 40), hit.index);
|
|
74203
|
+
const keyworded = ASOF_KEYWORDS.test(before) || ASOF_KEYWORDS.test(hit.match);
|
|
74204
|
+
const snippet = text.slice(Math.max(0, hit.index - 24), hit.index + hit.match.length + 4).replace(/\s+/g, " ").trim();
|
|
74205
|
+
out.push({
|
|
74206
|
+
date: hit.date,
|
|
74207
|
+
source: "content",
|
|
74208
|
+
confidence: keyworded ? 0.95 : 0.7,
|
|
74209
|
+
evidence: `${label}: "${snippet}"`
|
|
74210
|
+
});
|
|
74211
|
+
}
|
|
74212
|
+
return out;
|
|
74213
|
+
}
|
|
74214
|
+
function scanFilename(fileName) {
|
|
74215
|
+
if (!fileName) return [];
|
|
74216
|
+
const base = fileName.replace(/\.[A-Za-z0-9]+$/, "");
|
|
74217
|
+
return findDates(base).map((hit, i6, all) => ({
|
|
74218
|
+
date: hit.date,
|
|
74219
|
+
source: "filename",
|
|
74220
|
+
confidence: i6 === all.length - 1 ? 0.6 : 0.45,
|
|
74221
|
+
evidence: `file name: "${hit.match}"`
|
|
74222
|
+
}));
|
|
74223
|
+
}
|
|
74224
|
+
function detectAsOfCandidates(inputs) {
|
|
74225
|
+
const all = [];
|
|
74226
|
+
for (const t8 of inputs.texts ?? []) all.push(...scanText(t8.text, t8.label));
|
|
74227
|
+
if (inputs.fileName) all.push(...scanFilename(inputs.fileName));
|
|
74228
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
74229
|
+
for (const c6 of all) {
|
|
74230
|
+
const prev = byDate.get(c6.date);
|
|
74231
|
+
if (!prev || c6.confidence > prev.confidence) byDate.set(c6.date, c6);
|
|
74232
|
+
}
|
|
74233
|
+
return [...byDate.values()].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
74234
|
+
}
|
|
74235
|
+
function detectAsOf(fileName) {
|
|
74236
|
+
return scanFilename(fileName)[0]?.date ?? null;
|
|
74237
|
+
}
|
|
74238
|
+
|
|
74239
|
+
// src/import/materialize.ts
|
|
74240
|
+
function coerce2(v2, type) {
|
|
74241
|
+
if (v2 === null || v2 === void 0 || v2 === "") return null;
|
|
74242
|
+
if (type === "boolean") return v2 === true || v2 === "true" || v2 === 1 ? 1 : 0;
|
|
74243
|
+
return v2;
|
|
74244
|
+
}
|
|
74245
|
+
function contentKey(record) {
|
|
74246
|
+
const parts = Object.keys(record).sort().map((k6) => k6 + "=" + JSON.stringify(record[k6] ?? null));
|
|
74247
|
+
return createHash13("sha256").update(parts.join("|")).digest("hex");
|
|
74248
|
+
}
|
|
74249
|
+
function persistTable(configPath, name, fields) {
|
|
74250
|
+
if (!configPath || !existsSync25(configPath)) return;
|
|
74251
|
+
try {
|
|
74252
|
+
const doc = loadConfigDoc(configPath);
|
|
74253
|
+
doc.setIn(["entities", name], { fields, outputFile: name.toUpperCase() + ".md" });
|
|
74254
|
+
saveConfigDoc(configPath, doc);
|
|
74255
|
+
} catch {
|
|
74256
|
+
}
|
|
74257
|
+
}
|
|
74258
|
+
async function materializeImport(ctx, data, plan, views = [], opts = {}) {
|
|
74259
|
+
const { db, configPath } = ctx;
|
|
74260
|
+
const mode = opts.mode ?? "both";
|
|
74261
|
+
const doSchema = mode === "schema" || mode === "both";
|
|
74262
|
+
const doContents = mode === "contents" || mode === "both";
|
|
74263
|
+
const asOf = opts.asOf?.trim() ? opts.asOf.trim() : null;
|
|
74264
|
+
const asOfColumn = opts.asOfColumn?.trim() ? opts.asOfColumn.trim() : null;
|
|
74265
|
+
const dated = asOf !== null || asOfColumn !== null;
|
|
74266
|
+
const asOfSourceKey = (entity) => asOfColumn ? entity.columns.find((c6) => c6.name === asOfColumn)?.sourceKey ?? null : null;
|
|
74267
|
+
const rowAsOf = (entity, record) => {
|
|
74268
|
+
const sk = asOfSourceKey(entity);
|
|
74269
|
+
if (sk) {
|
|
74270
|
+
const d6 = parseCellDate(record[sk]);
|
|
74271
|
+
if (d6) return d6;
|
|
74272
|
+
}
|
|
74273
|
+
return asOf;
|
|
74274
|
+
};
|
|
74275
|
+
const recordKey = (entity, record) => {
|
|
74276
|
+
const a6 = rowAsOf(entity, record);
|
|
74277
|
+
return a6 ? contentKey({ ...record, __as_of: a6 }) : contentKey(record);
|
|
74278
|
+
};
|
|
74279
|
+
const scopedKey = (a6, keyVal) => (a6 ?? "") + "|" + normalizeText(keyVal);
|
|
74280
|
+
const report = async (p3) => {
|
|
74281
|
+
await opts.onProgress?.(p3);
|
|
74282
|
+
};
|
|
74283
|
+
const tablesCreated = [];
|
|
74284
|
+
const rowsByTable = {};
|
|
74285
|
+
const links = [];
|
|
74286
|
+
const viewResults = [];
|
|
74287
|
+
const byName = new Map(plan.entities.map((e6) => [e6.name, e6]));
|
|
74288
|
+
for (const entity of plan.entities) {
|
|
74289
|
+
const keyless = entity.naturalKey === null;
|
|
74290
|
+
const columns = { id: "TEXT PRIMARY KEY" };
|
|
74291
|
+
const fieldTypes = {};
|
|
74292
|
+
const cfgFields = { id: { type: "uuid", primaryKey: true } };
|
|
74293
|
+
for (const c6 of entity.columns) {
|
|
74294
|
+
columns[c6.name] = fieldToSqliteBaseType(c6.type);
|
|
74295
|
+
fieldTypes[c6.name] = c6.type;
|
|
74296
|
+
cfgFields[c6.name] = { type: c6.type };
|
|
74297
|
+
}
|
|
74298
|
+
const needsContentKey = keyless || dated;
|
|
74299
|
+
if (needsContentKey) {
|
|
74300
|
+
columns.content_key = "TEXT";
|
|
74301
|
+
cfgFields.content_key = { type: "text" };
|
|
74302
|
+
}
|
|
74303
|
+
if (dated) {
|
|
74304
|
+
columns.as_of = "TEXT";
|
|
74305
|
+
cfgFields.as_of = { type: "text" };
|
|
74306
|
+
}
|
|
74307
|
+
columns.deleted_at = "TEXT";
|
|
74308
|
+
cfgFields.deleted_at = { type: "text" };
|
|
74309
|
+
if (!db.getRegisteredTableNames().includes(entity.name)) tablesCreated.push(entity.name);
|
|
74310
|
+
await db.defineLate(entity.name, { columns, fieldTypes, primaryKey: "id" });
|
|
74311
|
+
persistTable(configPath, entity.name, cfgFields);
|
|
74312
|
+
await report({
|
|
74313
|
+
phase: "entities",
|
|
74314
|
+
table: entity.name,
|
|
74315
|
+
message: `Created table ${entity.name}`
|
|
74316
|
+
});
|
|
74317
|
+
if (doContents) {
|
|
74318
|
+
const records = sourceRecords(data, entity);
|
|
74319
|
+
const rows = records.map((r6) => {
|
|
74320
|
+
const row = {};
|
|
74321
|
+
for (const c6 of entity.columns) row[c6.name] = coerce2(r6[c6.sourceKey], c6.type);
|
|
74322
|
+
if (needsContentKey) row.content_key = recordKey(entity, r6);
|
|
74323
|
+
if (dated) row.as_of = rowAsOf(entity, r6);
|
|
74324
|
+
return row;
|
|
74325
|
+
});
|
|
74326
|
+
await db.seed({
|
|
74327
|
+
data: rows,
|
|
74328
|
+
table: entity.name,
|
|
74329
|
+
naturalKey: dated ? "content_key" : entity.naturalKey ?? "content_key"
|
|
74330
|
+
});
|
|
74331
|
+
const n3 = await db.count(entity.name);
|
|
74332
|
+
rowsByTable[entity.name] = n3;
|
|
74333
|
+
await report({
|
|
74334
|
+
phase: "entities",
|
|
74335
|
+
table: entity.name,
|
|
74336
|
+
count: n3,
|
|
74337
|
+
message: `Loaded ${String(n3)} rows into ${entity.name}`
|
|
74338
|
+
});
|
|
74339
|
+
}
|
|
74340
|
+
}
|
|
74341
|
+
for (const dim of plan.dimensions) {
|
|
74342
|
+
if (!db.getRegisteredTableNames().includes(dim.name)) tablesCreated.push(dim.name);
|
|
74343
|
+
await db.defineLate(dim.name, {
|
|
74344
|
+
columns: { id: "TEXT PRIMARY KEY", value: "TEXT", deleted_at: "TEXT" },
|
|
74345
|
+
fieldTypes: { value: "text" },
|
|
74346
|
+
primaryKey: "id"
|
|
74347
|
+
});
|
|
74348
|
+
persistTable(configPath, dim.name, {
|
|
74349
|
+
id: { type: "uuid", primaryKey: true },
|
|
74350
|
+
value: { type: "text" },
|
|
74351
|
+
deleted_at: { type: "text" }
|
|
74352
|
+
});
|
|
74353
|
+
if (doSchema) {
|
|
74354
|
+
const values = /* @__PURE__ */ new Map();
|
|
74355
|
+
for (const ename of dim.fromEntities) {
|
|
74356
|
+
const ent = byName.get(ename);
|
|
74357
|
+
if (!ent) continue;
|
|
74358
|
+
const records = sourceRecords(data, ent);
|
|
74359
|
+
const first = records[0];
|
|
74360
|
+
const srcKey = first ? Object.keys(first).find((k6) => normalizeName(k6) === dim.name) : void 0;
|
|
74361
|
+
if (!srcKey) continue;
|
|
74362
|
+
for (const r6 of records) {
|
|
74363
|
+
const v2 = r6[srcKey];
|
|
74364
|
+
if (typeof v2 !== "string" && typeof v2 !== "number") continue;
|
|
74365
|
+
const key = normalizeText(v2);
|
|
74366
|
+
if (key !== "" && !values.has(key)) values.set(key, String(v2));
|
|
74367
|
+
}
|
|
74368
|
+
}
|
|
74369
|
+
await db.seed({
|
|
74370
|
+
data: [...values.values()].map((value) => ({ value })),
|
|
74371
|
+
table: dim.name,
|
|
74372
|
+
naturalKey: "value"
|
|
74373
|
+
});
|
|
74374
|
+
const n3 = await db.count(dim.name);
|
|
74375
|
+
rowsByTable[dim.name] = n3;
|
|
74376
|
+
await report({
|
|
74377
|
+
phase: "dimensions",
|
|
74378
|
+
table: dim.name,
|
|
74379
|
+
count: n3,
|
|
74380
|
+
message: `Dimension ${dim.name}: ${String(n3)} values`
|
|
74381
|
+
});
|
|
74382
|
+
}
|
|
74383
|
+
}
|
|
74384
|
+
const idMapCache = /* @__PURE__ */ new Map();
|
|
74385
|
+
async function idMap(table, keyCol, datedTarget) {
|
|
74386
|
+
const cacheKey = table + ":" + keyCol + ":" + (datedTarget ? "D" : "");
|
|
74387
|
+
const cached = idMapCache.get(cacheKey);
|
|
74388
|
+
if (cached) return cached;
|
|
74389
|
+
const map = /* @__PURE__ */ new Map();
|
|
74390
|
+
for (const r6 of await db.query(table)) {
|
|
74391
|
+
const k6 = r6[keyCol];
|
|
74392
|
+
if (k6 === null || k6 === void 0) continue;
|
|
74393
|
+
const mapKey = datedTarget ? scopedKey(r6.as_of, k6) : normalizeText(k6);
|
|
74394
|
+
map.set(mapKey, String(r6.id));
|
|
74395
|
+
}
|
|
74396
|
+
idMapCache.set(cacheKey, map);
|
|
74397
|
+
return map;
|
|
74398
|
+
}
|
|
74399
|
+
for (const link of plan.linkages) {
|
|
74400
|
+
const from = byName.get(link.fromEntity);
|
|
74401
|
+
if (!from) continue;
|
|
74402
|
+
const jName = link.junction ?? `${link.fromEntity}_${link.toEntity}`;
|
|
74403
|
+
const fromFk = `${link.fromEntity}_id`;
|
|
74404
|
+
const toFk = `${link.toEntity}_id`;
|
|
74405
|
+
const jCols = {
|
|
74406
|
+
id: "TEXT PRIMARY KEY",
|
|
74407
|
+
[fromFk]: "TEXT",
|
|
74408
|
+
[toFk]: "TEXT"
|
|
74409
|
+
};
|
|
74410
|
+
const jCfg = {
|
|
74411
|
+
id: { type: "uuid", primaryKey: true },
|
|
74412
|
+
[fromFk]: { type: "uuid", ref: link.fromEntity },
|
|
74413
|
+
[toFk]: { type: "uuid", ref: link.toEntity }
|
|
74414
|
+
};
|
|
74415
|
+
if (dated) {
|
|
74416
|
+
jCols.as_of = "TEXT";
|
|
74417
|
+
jCfg.as_of = { type: "text" };
|
|
74418
|
+
}
|
|
74419
|
+
if (!db.getRegisteredTableNames().includes(jName)) tablesCreated.push(jName);
|
|
74420
|
+
await db.defineLate(jName, { columns: jCols, primaryKey: "id" });
|
|
74421
|
+
persistTable(configPath, jName, jCfg);
|
|
74422
|
+
if (!doContents) continue;
|
|
74423
|
+
const fromKeyCol = from.naturalKey ?? "content_key";
|
|
74424
|
+
const toIsEntity = byName.has(link.toEntity);
|
|
74425
|
+
const fromMap = await idMap(link.fromEntity, fromKeyCol, dated);
|
|
74426
|
+
const toMap = await idMap(link.toEntity, link.toKey, toIsEntity && dated);
|
|
74427
|
+
const seen = /* @__PURE__ */ new Set();
|
|
74428
|
+
for (const r6 of await db.query(jName)) {
|
|
74429
|
+
seen.add(String(r6[fromFk]) + "|" + String(r6[toFk]));
|
|
74430
|
+
}
|
|
74431
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
74432
|
+
let created = 0;
|
|
74433
|
+
for (const record of sourceRecords(data, from)) {
|
|
74434
|
+
const a6 = rowAsOf(from, record);
|
|
74435
|
+
const fromKeyVal = from.naturalKey === null ? recordKey(from, record) : record[from.naturalKeySource ?? ""];
|
|
74436
|
+
const fromId = fromMap.get(dated ? scopedKey(a6, fromKeyVal) : normalizeText(fromKeyVal));
|
|
74437
|
+
if (!fromId) continue;
|
|
74438
|
+
const raw = record[link.fromField];
|
|
74439
|
+
const refs = Array.isArray(raw) ? raw : [raw];
|
|
74440
|
+
for (const ref of refs) {
|
|
74441
|
+
if (ref === null || ref === void 0 || ref === "") continue;
|
|
74442
|
+
const toId = toMap.get(toIsEntity && dated ? scopedKey(a6, ref) : normalizeText(ref));
|
|
74443
|
+
if (!toId) {
|
|
74444
|
+
unresolved.add(normalizeText(ref));
|
|
74445
|
+
continue;
|
|
74446
|
+
}
|
|
74447
|
+
const edge = fromId + "|" + toId;
|
|
74448
|
+
if (seen.has(edge)) continue;
|
|
74449
|
+
seen.add(edge);
|
|
74450
|
+
await db.insert(
|
|
74451
|
+
jName,
|
|
74452
|
+
dated ? { [fromFk]: fromId, [toFk]: toId, as_of: a6 } : { [fromFk]: fromId, [toFk]: toId }
|
|
74453
|
+
);
|
|
74454
|
+
created++;
|
|
74455
|
+
}
|
|
74456
|
+
}
|
|
74457
|
+
rowsByTable[jName] = created;
|
|
74458
|
+
links.push({ junction: jName, created, unresolved: unresolved.size });
|
|
74459
|
+
await report({
|
|
74460
|
+
phase: "links",
|
|
74461
|
+
table: jName,
|
|
74462
|
+
count: created,
|
|
74463
|
+
message: `Linked ${String(created)} ${jName}`
|
|
74464
|
+
});
|
|
74465
|
+
}
|
|
74466
|
+
if (doSchema) {
|
|
74467
|
+
for (const v2 of views) {
|
|
74468
|
+
const filt = v2.filterValue.replace(/'/g, "''");
|
|
74469
|
+
await execSql(db, `DROP VIEW IF EXISTS "${v2.name}"`);
|
|
74470
|
+
await execSql(
|
|
74471
|
+
db,
|
|
74472
|
+
`CREATE VIEW "${v2.name}" AS SELECT * FROM "${v2.master}" WHERE "${v2.filterColumn}" = '${filt}'`
|
|
74473
|
+
);
|
|
74474
|
+
const cols = await db.introspectColumns(v2.name);
|
|
74475
|
+
await db.defineLate(v2.name, {
|
|
74476
|
+
columns: Object.fromEntries(cols.map((c6) => [c6, "TEXT"])),
|
|
74477
|
+
render: () => ""
|
|
74478
|
+
});
|
|
74479
|
+
if (!tablesCreated.includes(v2.name)) tablesCreated.push(v2.name);
|
|
74480
|
+
const rows = await db.count(v2.name);
|
|
74481
|
+
rowsByTable[v2.name] = rows;
|
|
74482
|
+
viewResults.push({ name: v2.name, master: v2.master, rows });
|
|
74483
|
+
await report({
|
|
74484
|
+
phase: "views",
|
|
74485
|
+
table: v2.name,
|
|
74486
|
+
count: rows,
|
|
74487
|
+
message: `View ${v2.name}: ${String(rows)} rows`
|
|
74488
|
+
});
|
|
74489
|
+
}
|
|
74490
|
+
}
|
|
74491
|
+
await report({ phase: "done", message: "Import complete" });
|
|
74492
|
+
return { mode, asOf, asOfColumn, tablesCreated, rowsByTable, links, views: viewResults };
|
|
74493
|
+
}
|
|
74494
|
+
|
|
74495
|
+
// src/gui/import-auto.ts
|
|
74496
|
+
init_native_entities();
|
|
74497
|
+
|
|
74498
|
+
// src/gui/import-detect.ts
|
|
74499
|
+
import { basename as basename11 } from "path";
|
|
74500
|
+
|
|
74501
|
+
// src/gui/ai/asof-llm.ts
|
|
74502
|
+
init_assistant_routes();
|
|
74503
|
+
init_chat();
|
|
74504
|
+
var MAX_CHARS = 6e3;
|
|
74505
|
+
var SYSTEM = 'You extract the single "as of" / report / snapshot / period-end date from the text of a data file (a financial statement, track record, export, etc.). Reply with ONLY that date as ISO YYYY-MM-DD, or the exact word NONE if the text has no such date. Output nothing else \u2014 no prose, no quotes.';
|
|
74506
|
+
function parseLlmDate(reply) {
|
|
74507
|
+
if (!reply) return null;
|
|
74508
|
+
const m4 = /(20\d{2})-(\d{2})-(\d{2})/.exec(reply);
|
|
74509
|
+
if (!m4) return null;
|
|
74510
|
+
const y2 = Number(m4[1]);
|
|
74511
|
+
const mo = Number(m4[2]);
|
|
74512
|
+
const d6 = Number(m4[3]);
|
|
74513
|
+
if (mo < 1 || mo > 12 || d6 < 1 || d6 > 31 || y2 < 2010 || y2 > 2099) return null;
|
|
74514
|
+
return `${String(y2)}-${String(mo).padStart(2, "0")}-${String(d6).padStart(2, "0")}`;
|
|
74515
|
+
}
|
|
74516
|
+
async function asOfFromLlm(db, text) {
|
|
74517
|
+
const trimmed = text.trim();
|
|
74518
|
+
if (!trimmed) return null;
|
|
74519
|
+
try {
|
|
74520
|
+
const auth = await resolveClaudeAuth(db);
|
|
74521
|
+
if (!auth) return null;
|
|
74522
|
+
const client = createAnthropicClient(auth);
|
|
74523
|
+
const result = await client.runTurn({
|
|
74524
|
+
model: DEFAULT_MODEL2,
|
|
74525
|
+
system: SYSTEM,
|
|
74526
|
+
temperature: 0,
|
|
74527
|
+
tools: [],
|
|
74528
|
+
messages: [{ role: "user", content: `File text:
|
|
74529
|
+
${trimmed.slice(0, MAX_CHARS)}` }],
|
|
74530
|
+
onText: () => {
|
|
74531
|
+
}
|
|
74532
|
+
});
|
|
74533
|
+
const date2 = parseLlmDate(result.text);
|
|
74534
|
+
return date2 ? { date: date2, source: "llm", confidence: 0.85, evidence: "Claude read the file" } : null;
|
|
74535
|
+
} catch (e6) {
|
|
74536
|
+
console.warn("[import] as-of LLM fallback failed:", e6.message);
|
|
74537
|
+
return null;
|
|
74538
|
+
}
|
|
74539
|
+
}
|
|
74540
|
+
|
|
74541
|
+
// src/gui/import-detect.ts
|
|
74542
|
+
async function detectImportAsOf(db, data, opts = {}) {
|
|
74543
|
+
const fileName = opts.fileName ?? (opts.abs ? basename11(opts.abs).replace(/^[0-9a-f]{8}-/, "") : "");
|
|
74544
|
+
const texts = [];
|
|
74545
|
+
for (const [k6, v2] of Object.entries(data)) {
|
|
74546
|
+
if (!Array.isArray(v2)) texts.push({ label: "data", text: `${k6}: ${JSON.stringify(v2)}` });
|
|
74547
|
+
}
|
|
74548
|
+
if (opts.abs && /\.xlsx?$/i.test(opts.abs)) {
|
|
74549
|
+
const pre = excelPreambleText(opts.abs);
|
|
74550
|
+
if (pre) texts.push({ label: "title", text: pre });
|
|
74551
|
+
}
|
|
74552
|
+
let candidates = detectAsOfCandidates({ fileName, texts });
|
|
74553
|
+
if (!candidates[0] || candidates[0].confidence < 0.7) {
|
|
74554
|
+
const llm = await asOfFromLlm(db, texts.map((t8) => t8.text).join("\n"));
|
|
74555
|
+
if (llm) candidates = [...candidates, llm].sort((a6, b6) => b6.confidence - a6.confidence);
|
|
74556
|
+
}
|
|
74557
|
+
return candidates;
|
|
74558
|
+
}
|
|
74559
|
+
|
|
74560
|
+
// src/import/asof-columns.ts
|
|
74561
|
+
var STRONG_NAME = /(as[_ -]?of|as[_ -]?at|report(?:ing)?[_ -]?date|valuation[_ -]?date|effective[_ -]?date|period[_ -]?end|snapshot[_ -]?date|statement[_ -]?date|fye)/i;
|
|
74562
|
+
var WEAK_NAME = /(^|_)(date|period|quarter|asof)($|_)/i;
|
|
74563
|
+
function detectAsOfColumns(data, plan) {
|
|
74564
|
+
const out = [];
|
|
74565
|
+
for (const entity of plan.entities) {
|
|
74566
|
+
const records = sourceRecords(data, entity);
|
|
74567
|
+
if (records.length < 2) continue;
|
|
74568
|
+
for (const col of entity.columns) {
|
|
74569
|
+
const strong = STRONG_NAME.test(col.name);
|
|
74570
|
+
const weak = WEAK_NAME.test(col.name);
|
|
74571
|
+
if (!strong && !weak) continue;
|
|
74572
|
+
const vals = records.map((r6) => r6[col.sourceKey]).filter((v2) => v2 !== null && v2 !== void 0 && v2 !== "");
|
|
74573
|
+
if (vals.length < Math.max(3, Math.floor(records.length * 0.5))) continue;
|
|
74574
|
+
const dates = vals.map(parseCellDate).filter((d6) => d6 !== null);
|
|
74575
|
+
if (dates.length / vals.length < 0.8) continue;
|
|
74576
|
+
const distinctDates = new Set(dates).size;
|
|
74577
|
+
const typed = col.type === "date" || col.type === "datetime";
|
|
74578
|
+
let confidence = strong ? 0.9 : 0.6;
|
|
74579
|
+
if (typed) confidence += 0.03;
|
|
74580
|
+
if (distinctDates > 1) confidence += 0.04;
|
|
74581
|
+
out.push({
|
|
74582
|
+
entity: entity.name,
|
|
74583
|
+
column: col.name,
|
|
74584
|
+
confidence: Math.min(confidence, 0.97),
|
|
74585
|
+
distinctDates,
|
|
74586
|
+
evidence: `column "${col.name}" \u2014 ${String(distinctDates)} distinct date${distinctDates === 1 ? "" : "s"} across ${String(vals.length)} rows`
|
|
74587
|
+
});
|
|
74588
|
+
}
|
|
74589
|
+
}
|
|
74590
|
+
return out.sort((a6, b6) => b6.confidence - a6.confidence);
|
|
74591
|
+
}
|
|
74592
|
+
|
|
74593
|
+
// src/gui/import-auto.ts
|
|
74594
|
+
function existingDataTables(db) {
|
|
74595
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
74596
|
+
const out = [];
|
|
74597
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
74598
|
+
if (native.has(t8)) continue;
|
|
74599
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
74600
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
74601
|
+
}
|
|
74602
|
+
return out;
|
|
74603
|
+
}
|
|
74604
|
+
async function readStructured(abs, name) {
|
|
74605
|
+
if (/\.xlsx?$/i.test(name)) return excelToRecords(abs);
|
|
74606
|
+
return JSON.parse(readFileSync23(abs, "utf8"));
|
|
74607
|
+
}
|
|
74608
|
+
async function autoImportStructured(db, configPath, abs, name) {
|
|
74609
|
+
if (!/\.(xlsx?|json)$/i.test(name)) return null;
|
|
74610
|
+
let data;
|
|
74611
|
+
try {
|
|
74612
|
+
data = await readStructured(abs, name);
|
|
74613
|
+
} catch {
|
|
74614
|
+
return null;
|
|
74615
|
+
}
|
|
74616
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
74617
|
+
inferSchema(data),
|
|
74618
|
+
data
|
|
74619
|
+
);
|
|
74620
|
+
if (inferredPlan.entities.length === 0) return null;
|
|
74621
|
+
const schemaMatch = matchSchemaToExisting(existingDataTables(db), inferredPlan);
|
|
74622
|
+
const asOfCandidates = await detectImportAsOf(db, data, { abs, fileName: name });
|
|
74623
|
+
const asOf = asOfCandidates[0]?.date ?? null;
|
|
74624
|
+
const asOfColumns = detectAsOfColumns(data, inferredPlan);
|
|
74625
|
+
const proposal = {
|
|
74626
|
+
plan: inferredPlan,
|
|
74627
|
+
views: inferredViews,
|
|
74628
|
+
asOfCandidates,
|
|
74629
|
+
asOfColumns,
|
|
74630
|
+
schemaMatch,
|
|
74631
|
+
matchedCount: schemaMatch.matchedCount,
|
|
74632
|
+
totalEntities: schemaMatch.totalEntities,
|
|
74633
|
+
tables: [],
|
|
74634
|
+
rows: 0
|
|
74635
|
+
};
|
|
74636
|
+
if (!schemaMatch.isKnownDocument) {
|
|
74637
|
+
return { imported: false, reason: "new-dataset", asOf, ...proposal };
|
|
74638
|
+
}
|
|
74639
|
+
if (!asOf) {
|
|
74640
|
+
return { imported: false, reason: "needs-confirm", asOf: null, ...proposal };
|
|
74641
|
+
}
|
|
74642
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, schemaMatch.rename);
|
|
74643
|
+
const result = await materializeImport({ db, configPath }, data, plan, views, { asOf });
|
|
74644
|
+
const rows = Object.values(result.rowsByTable).reduce((a6, b6) => a6 + b6, 0);
|
|
74645
|
+
return {
|
|
74646
|
+
imported: true,
|
|
74647
|
+
asOf,
|
|
74648
|
+
matchedCount: schemaMatch.matchedCount,
|
|
74649
|
+
totalEntities: schemaMatch.totalEntities,
|
|
74650
|
+
tables: Object.keys(result.rowsByTable),
|
|
74651
|
+
rows
|
|
74652
|
+
};
|
|
74653
|
+
}
|
|
74654
|
+
|
|
74655
|
+
// src/gui/ingest-routes.ts
|
|
72963
74656
|
var MIME_BY_EXT = {
|
|
72964
74657
|
".pdf": "application/pdf",
|
|
72965
74658
|
".png": "image/png",
|
|
@@ -73061,28 +74754,28 @@ ${err.stack ?? ""}`
|
|
|
73061
74754
|
return null;
|
|
73062
74755
|
}
|
|
73063
74756
|
}
|
|
73064
|
-
async function extractImage(db,
|
|
74757
|
+
async function extractImage(db, path3, mime) {
|
|
73065
74758
|
if (!mime.startsWith("image/")) return null;
|
|
73066
74759
|
const auth = await resolveClaudeAuth(db);
|
|
73067
74760
|
if (!auth) return null;
|
|
73068
74761
|
try {
|
|
73069
|
-
const text = await describeImage(auth,
|
|
74762
|
+
const text = await describeImage(auth, path3);
|
|
73070
74763
|
return text.trim() ? { text, skip: false } : null;
|
|
73071
74764
|
} catch (e6) {
|
|
73072
74765
|
console.warn("[ingest] image vision failed:", e6.message);
|
|
73073
74766
|
return null;
|
|
73074
74767
|
}
|
|
73075
74768
|
}
|
|
73076
|
-
async function extractSource(db,
|
|
73077
|
-
const vision = await extractImage(db,
|
|
74769
|
+
async function extractSource(db, path3, mime, name) {
|
|
74770
|
+
const vision = await extractImage(db, path3, mime);
|
|
73078
74771
|
if (vision) return vision;
|
|
73079
|
-
const parsed = await parseFile(
|
|
74772
|
+
const parsed = await parseFile(path3, mime, name);
|
|
73080
74773
|
if (!parsed.skip) return parsed;
|
|
73081
74774
|
if (mime === "application/pdf") {
|
|
73082
74775
|
const auth = await resolveClaudeAuth(db);
|
|
73083
74776
|
if (auth) {
|
|
73084
74777
|
try {
|
|
73085
|
-
const text = await describePdf(auth,
|
|
74778
|
+
const text = await describePdf(auth, path3);
|
|
73086
74779
|
if (text.trim()) return { ...parsed, text, skip: false };
|
|
73087
74780
|
} catch (e6) {
|
|
73088
74781
|
console.warn("[ingest] Claude PDF read failed:", e6.message);
|
|
@@ -73095,7 +74788,6 @@ function looksLikeUrl(s2) {
|
|
|
73095
74788
|
const t8 = s2.trim();
|
|
73096
74789
|
return /^https?:\/\/\S+$/i.test(t8) && !/\s/.test(t8);
|
|
73097
74790
|
}
|
|
73098
|
-
var MAX_INGEST_BYTES = 5e7;
|
|
73099
74791
|
function readBuffer2(req, maxBytes = MAX_INGEST_BYTES) {
|
|
73100
74792
|
return new Promise((resolve_, reject) => {
|
|
73101
74793
|
const chunks = [];
|
|
@@ -73157,9 +74849,15 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73157
74849
|
const tmp = join29(tmpdir2(), `lattice-ingest-${crypto.randomUUID()}${extname2(name2)}`);
|
|
73158
74850
|
let result;
|
|
73159
74851
|
let blob = null;
|
|
74852
|
+
let autoImport = null;
|
|
73160
74853
|
try {
|
|
73161
74854
|
await writeFile2(tmp, buf);
|
|
73162
74855
|
result = await extractSource(ctx.db, tmp, mime2, name2);
|
|
74856
|
+
try {
|
|
74857
|
+
autoImport = await autoImportStructured(ctx.db, ctx.configPath ?? null, tmp, name2);
|
|
74858
|
+
} catch (e6) {
|
|
74859
|
+
console.warn("[ingest] auto-import skipped:", e6.message);
|
|
74860
|
+
}
|
|
73163
74861
|
if (ctx.latticeRoot && !realPath && shouldRetainUploadBlob(mime2, name2)) {
|
|
73164
74862
|
try {
|
|
73165
74863
|
const meta2 = await attachBlob(tmp, ctx.latticeRoot);
|
|
@@ -73175,7 +74873,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73175
74873
|
let s3Status = null;
|
|
73176
74874
|
const s3cfg = resolveActiveS3Config(ctx.configPath);
|
|
73177
74875
|
if (s3cfg) {
|
|
73178
|
-
const sha256 = blob?.sha256 ??
|
|
74876
|
+
const sha256 = blob?.sha256 ?? createHash14("sha256").update(buf).digest("hex");
|
|
73179
74877
|
const key = s3Key(s3cfg.prefix, sha256);
|
|
73180
74878
|
try {
|
|
73181
74879
|
const store = await createS3Store(s3cfg);
|
|
@@ -73198,7 +74896,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73198
74896
|
}
|
|
73199
74897
|
}
|
|
73200
74898
|
const fileId = crypto.randomUUID();
|
|
73201
|
-
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ??
|
|
74899
|
+
const fileSha = blob?.sha256 ?? s3Ref?.sha256 ?? createHash14("sha256").update(buf).digest("hex");
|
|
73202
74900
|
const uploadRow = {
|
|
73203
74901
|
id: fileId,
|
|
73204
74902
|
...fileIdentity(name2, fileId),
|
|
@@ -73236,6 +74934,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73236
74934
|
},
|
|
73237
74935
|
forcePrivate2 ? "private" : void 0
|
|
73238
74936
|
);
|
|
74937
|
+
if (autoImport?.reason) autoImport.fileId = id2;
|
|
73239
74938
|
try {
|
|
73240
74939
|
const dedupCtx = {
|
|
73241
74940
|
db: ctx.db,
|
|
@@ -73263,6 +74962,15 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73263
74962
|
e6 instanceof Error ? e6.message : String(e6)
|
|
73264
74963
|
);
|
|
73265
74964
|
}
|
|
74965
|
+
if (autoImport?.imported) {
|
|
74966
|
+
ctx.feed.publish({
|
|
74967
|
+
table: autoImport.tables[0] ?? "files",
|
|
74968
|
+
op: "insert",
|
|
74969
|
+
rowId: null,
|
|
74970
|
+
source: "system",
|
|
74971
|
+
summary: `Imported the ${autoImport.asOf ?? ""} snapshot of "${name2}" \u2014 ${String(autoImport.rows)} rows across ${String(autoImport.tables.length)} tables`
|
|
74972
|
+
});
|
|
74973
|
+
}
|
|
73266
74974
|
let suggestedLinks = [];
|
|
73267
74975
|
if (!result.skip) {
|
|
73268
74976
|
const links = await enrichOrFail(mctx, ctx.db, id2, result.text, name2, ctx, res, forcePrivate2);
|
|
@@ -73275,6 +74983,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73275
74983
|
id: id2,
|
|
73276
74984
|
extraction_status: result.skip ? "skipped" : "extracted",
|
|
73277
74985
|
suggestedLinks,
|
|
74986
|
+
...autoImport ? { autoImport } : {},
|
|
73278
74987
|
// Present only when S3 is enabled for this workspace. 'failed' tells the
|
|
73279
74988
|
// uploader the bytes did NOT reach the shared bucket — other members would
|
|
73280
74989
|
// 404 until it's re-uploaded — so the GUI can warn rather than imply a
|
|
@@ -73360,7 +75069,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73360
75069
|
sendJson(res, { error: "path is required" }, 400);
|
|
73361
75070
|
return true;
|
|
73362
75071
|
}
|
|
73363
|
-
const abs =
|
|
75072
|
+
const abs = resolve11(rawPath);
|
|
73364
75073
|
let size = 0;
|
|
73365
75074
|
try {
|
|
73366
75075
|
const st = statSync9(abs);
|
|
@@ -73377,7 +75086,7 @@ async function dispatchIngestRoute(req, res, ctx) {
|
|
|
73377
75086
|
sendJson(res, { error: "file too large" }, 413);
|
|
73378
75087
|
return true;
|
|
73379
75088
|
}
|
|
73380
|
-
const name =
|
|
75089
|
+
const name = basename12(abs);
|
|
73381
75090
|
const mime = mimeFor(name);
|
|
73382
75091
|
const localFileId = crypto.randomUUID();
|
|
73383
75092
|
const localRow = {
|
|
@@ -73443,6 +75152,146 @@ ${err.stack ?? ""}`
|
|
|
73443
75152
|
return true;
|
|
73444
75153
|
}
|
|
73445
75154
|
|
|
75155
|
+
// src/gui/import-routes.ts
|
|
75156
|
+
init_adapter();
|
|
75157
|
+
init_http2();
|
|
75158
|
+
import { existsSync as existsSync26, readFileSync as readFileSync24, statSync as statSync10 } from "fs";
|
|
75159
|
+
import { isAbsolute as isAbsolute4, join as join30 } from "path";
|
|
75160
|
+
init_native_entities();
|
|
75161
|
+
function badRequest(message) {
|
|
75162
|
+
const e6 = new Error(message);
|
|
75163
|
+
e6.statusCode = 400;
|
|
75164
|
+
return e6;
|
|
75165
|
+
}
|
|
75166
|
+
function localPathOf2(row, latticeRoot) {
|
|
75167
|
+
if (row.ref_kind === "local_ref" && row.ref_uri) return row.ref_uri;
|
|
75168
|
+
if ((row.ref_kind === "blob" || row.ref_kind === "cloud_ref") && row.blob_path) {
|
|
75169
|
+
return isAbsolute4(row.blob_path) ? row.blob_path : latticeRoot ? join30(latticeRoot, row.blob_path) : null;
|
|
75170
|
+
}
|
|
75171
|
+
return null;
|
|
75172
|
+
}
|
|
75173
|
+
function existingDataTables2(db) {
|
|
75174
|
+
const native = new Set(NATIVE_ENTITY_NAMES);
|
|
75175
|
+
const out = [];
|
|
75176
|
+
for (const t8 of db.getRegisteredTableNames()) {
|
|
75177
|
+
if (native.has(t8)) continue;
|
|
75178
|
+
const columns = Object.keys(db.getRegisteredColumns(t8) ?? {});
|
|
75179
|
+
if (columns.length > 0) out.push({ name: t8, columns });
|
|
75180
|
+
}
|
|
75181
|
+
return out;
|
|
75182
|
+
}
|
|
75183
|
+
async function readImportSourceFromFile(db, fileId, latticeRoot) {
|
|
75184
|
+
const row = await getAsyncOrSync(
|
|
75185
|
+
db.adapter,
|
|
75186
|
+
`SELECT "id","original_name","mime","ref_kind","ref_uri","blob_path"
|
|
75187
|
+
FROM "files" WHERE "id" = ? AND "deleted_at" IS NULL LIMIT 1`,
|
|
75188
|
+
[fileId]
|
|
75189
|
+
);
|
|
75190
|
+
if (!row) throw badRequest("Unknown import file: " + fileId);
|
|
75191
|
+
const path3 = localPathOf2(row, latticeRoot);
|
|
75192
|
+
if (!path3 || !existsSync26(path3)) {
|
|
75193
|
+
throw badRequest("The import file\u2019s bytes are not available locally.");
|
|
75194
|
+
}
|
|
75195
|
+
const sizeBytes = statSync10(path3).size;
|
|
75196
|
+
if (sizeBytes > MAX_INGEST_BYTES) {
|
|
75197
|
+
throw badRequest(
|
|
75198
|
+
`The import file is too large (${String(Math.round(sizeBytes / 1e6))} MB); the limit is ${String(Math.round(MAX_INGEST_BYTES / 1e6))} MB.`
|
|
75199
|
+
);
|
|
75200
|
+
}
|
|
75201
|
+
const name = row.original_name ?? "";
|
|
75202
|
+
const mime = row.mime ?? "";
|
|
75203
|
+
if (/\.xlsx?$/i.test(name) || mime.includes("spreadsheet") || mime.includes("excel")) {
|
|
75204
|
+
return excelToRecords(path3);
|
|
75205
|
+
}
|
|
75206
|
+
let parsed;
|
|
75207
|
+
try {
|
|
75208
|
+
parsed = JSON.parse(readFileSync24(path3, "utf8"));
|
|
75209
|
+
} catch {
|
|
75210
|
+
throw badRequest("The import file is not valid JSON.");
|
|
75211
|
+
}
|
|
75212
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
75213
|
+
throw badRequest("Expected a JSON object whose keys are record arrays.");
|
|
75214
|
+
}
|
|
75215
|
+
return parsed;
|
|
75216
|
+
}
|
|
75217
|
+
async function dispatchImportRoute(req, res, deps) {
|
|
75218
|
+
const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
|
|
75219
|
+
if (req.method !== "POST" || pathname !== "/api/import/apply") return false;
|
|
75220
|
+
const body = await readJson(req).catch(() => ({}));
|
|
75221
|
+
const fileId = typeof body.fileId === "string" ? body.fileId : "";
|
|
75222
|
+
const mode = body.mode === "schema" || body.mode === "contents" ? body.mode : "both";
|
|
75223
|
+
const asOf = typeof body.asOf === "string" && /^\d{4}-\d{2}-\d{2}$/.test(body.asOf.trim()) ? body.asOf.trim() : null;
|
|
75224
|
+
const asOfColumn = typeof body.asOfColumn === "string" && body.asOfColumn.trim() ? body.asOfColumn.trim() : null;
|
|
75225
|
+
if (!fileId) {
|
|
75226
|
+
sendJson(res, { error: "fileId is required" }, 400);
|
|
75227
|
+
return true;
|
|
75228
|
+
}
|
|
75229
|
+
res.writeHead(200, {
|
|
75230
|
+
"content-type": "application/x-ndjson; charset=utf-8",
|
|
75231
|
+
"cache-control": "no-store"
|
|
75232
|
+
});
|
|
75233
|
+
const emit = (p3) => {
|
|
75234
|
+
res.write(JSON.stringify(p3) + "\n");
|
|
75235
|
+
};
|
|
75236
|
+
try {
|
|
75237
|
+
emit({ phase: "parse", message: "Reading source\u2026" });
|
|
75238
|
+
const data = await readImportSourceFromFile(deps.db, fileId, deps.latticeRoot);
|
|
75239
|
+
emit({ phase: "infer", message: "Analyzing schema\u2026" });
|
|
75240
|
+
const { plan: inferredPlan, views: inferredViews } = dedupeAndDetectViews(
|
|
75241
|
+
inferSchema(data),
|
|
75242
|
+
data
|
|
75243
|
+
);
|
|
75244
|
+
emit({
|
|
75245
|
+
phase: "infer",
|
|
75246
|
+
message: `Found ${String(inferredPlan.entities.length)} entities, ${String(inferredPlan.dimensions.length)} dimensions, ${String(inferredPlan.linkages.length)} links`
|
|
75247
|
+
});
|
|
75248
|
+
const match = matchSchemaToExisting(existingDataTables2(deps.db), inferredPlan);
|
|
75249
|
+
const { plan, views } = renameEntities(inferredPlan, inferredViews, match.rename);
|
|
75250
|
+
if (views.length > 0) {
|
|
75251
|
+
emit({
|
|
75252
|
+
phase: "detect",
|
|
75253
|
+
message: `Detected ${String(views.length)} reconstructable views (no duplicated rows)`
|
|
75254
|
+
});
|
|
75255
|
+
}
|
|
75256
|
+
if (match.isKnownDocument) {
|
|
75257
|
+
emit({
|
|
75258
|
+
phase: "detect",
|
|
75259
|
+
message: `Recognized as a new period of an existing document \u2014 ${String(match.matchedCount)} of ${String(match.totalEntities)} tables matched`
|
|
75260
|
+
});
|
|
75261
|
+
}
|
|
75262
|
+
if (asOfColumn) {
|
|
75263
|
+
emit({ phase: "infer", message: `Dating each row by its "${asOfColumn}" column` });
|
|
75264
|
+
} else if (asOf) {
|
|
75265
|
+
emit({ phase: "infer", message: `Importing as a snapshot dated ${asOf}` });
|
|
75266
|
+
}
|
|
75267
|
+
const result = await materializeImport(
|
|
75268
|
+
{ db: deps.db, configPath: deps.configPath },
|
|
75269
|
+
data,
|
|
75270
|
+
plan,
|
|
75271
|
+
views,
|
|
75272
|
+
{
|
|
75273
|
+
mode,
|
|
75274
|
+
asOf,
|
|
75275
|
+
asOfColumn,
|
|
75276
|
+
onProgress: async (p3) => {
|
|
75277
|
+
emit({ ...p3 });
|
|
75278
|
+
await new Promise((r6) => setImmediate(r6));
|
|
75279
|
+
}
|
|
75280
|
+
}
|
|
75281
|
+
);
|
|
75282
|
+
for (const t8 of result.tablesCreated) {
|
|
75283
|
+
deps.validTables.add(t8);
|
|
75284
|
+
const cols = deps.db.getRegisteredColumns(t8);
|
|
75285
|
+
if (cols && "deleted_at" in cols) deps.softDeletable.add(t8);
|
|
75286
|
+
}
|
|
75287
|
+
emit({ phase: "done", ok: true, result });
|
|
75288
|
+
} catch (e6) {
|
|
75289
|
+
emit({ phase: "error", message: e6.message });
|
|
75290
|
+
}
|
|
75291
|
+
res.end();
|
|
75292
|
+
return true;
|
|
75293
|
+
}
|
|
75294
|
+
|
|
73446
75295
|
// src/gui/read-routes.ts
|
|
73447
75296
|
init_http2();
|
|
73448
75297
|
init_data();
|
|
@@ -73731,7 +75580,13 @@ async function handleReadRoutes(req, res, ctx, deps) {
|
|
|
73731
75580
|
return true;
|
|
73732
75581
|
}
|
|
73733
75582
|
if (method === "GET" && pathname === "/api/history") {
|
|
73734
|
-
const
|
|
75583
|
+
const limitRaw = url.searchParams.get("limit");
|
|
75584
|
+
const parsedLimit = parsePageParam(limitRaw, "limit");
|
|
75585
|
+
if (parsedLimit === "invalid") {
|
|
75586
|
+
sendJson(res, { error: "limit must be a non-negative integer" }, 400);
|
|
75587
|
+
return true;
|
|
75588
|
+
}
|
|
75589
|
+
const limit = limitRaw === null ? 200 : parsedLimit;
|
|
73735
75590
|
const filterTable = url.searchParams.get("table");
|
|
73736
75591
|
const raw = await active.db.query("_lattice_gui_audit", { limit });
|
|
73737
75592
|
let entries = raw.map(parseAudit).sort((a6, b6) => b6.ts.localeCompare(a6.ts));
|
|
@@ -74775,8 +76630,8 @@ async function handleHistoryRoutes(req, res, ctx, deps) {
|
|
|
74775
76630
|
|
|
74776
76631
|
// src/gui/workspaces-routes.ts
|
|
74777
76632
|
init_http2();
|
|
74778
|
-
import { resolve as
|
|
74779
|
-
import { existsSync as
|
|
76633
|
+
import { resolve as resolve12 } from "path";
|
|
76634
|
+
import { existsSync as existsSync27, rmSync } from "fs";
|
|
74780
76635
|
init_workspace();
|
|
74781
76636
|
init_lattice_root();
|
|
74782
76637
|
init_user_config();
|
|
@@ -74784,7 +76639,7 @@ function cleanupWorkspaceFiles(root6, ws) {
|
|
|
74784
76639
|
if (!ws.configPath && ws.kind === "local") {
|
|
74785
76640
|
rmSync(workspaceDir(root6, ws.dir), { recursive: true, force: true });
|
|
74786
76641
|
} else if (ws.kind === "cloud") {
|
|
74787
|
-
if (ws.configPath &&
|
|
76642
|
+
if (ws.configPath && existsSync27(ws.configPath)) {
|
|
74788
76643
|
rmSync(ws.configPath, { force: true });
|
|
74789
76644
|
}
|
|
74790
76645
|
const labelMatch = /^\$\{LATTICE_DB:([A-Za-z0-9._-]+)\}$/.exec(ws.db.trim());
|
|
@@ -74938,7 +76793,7 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
74938
76793
|
return true;
|
|
74939
76794
|
}
|
|
74940
76795
|
const wsPaths = resolveWorkspacePaths(latticeRoot, ws);
|
|
74941
|
-
const isActive =
|
|
76796
|
+
const isActive = resolve12(active.configPath) === resolve12(wsPaths.configPath);
|
|
74942
76797
|
let switchedTo = null;
|
|
74943
76798
|
if (isActive) {
|
|
74944
76799
|
const fallback = listWorkspaces(latticeRoot).find((w2) => w2.id !== ws.id);
|
|
@@ -74987,40 +76842,40 @@ async function handleWorkspacesRoutes(req, res, ctx, deps) {
|
|
|
74987
76842
|
|
|
74988
76843
|
// src/gui/databases-routes.ts
|
|
74989
76844
|
init_http2();
|
|
74990
|
-
import { basename as
|
|
74991
|
-
import { existsSync as
|
|
76845
|
+
import { basename as basename14, resolve as resolve14 } from "path";
|
|
76846
|
+
import { existsSync as existsSync29 } from "fs";
|
|
74992
76847
|
init_parser();
|
|
74993
76848
|
|
|
74994
76849
|
// src/gui/config-paths.ts
|
|
74995
76850
|
init_parser();
|
|
74996
|
-
import { basename as
|
|
76851
|
+
import { basename as basename13, dirname as dirname15, join as join31, resolve as resolve13 } from "path";
|
|
74997
76852
|
import {
|
|
74998
|
-
existsSync as
|
|
76853
|
+
existsSync as existsSync28,
|
|
74999
76854
|
mkdirSync as mkdirSync11,
|
|
75000
|
-
readFileSync as
|
|
76855
|
+
readFileSync as readFileSync25,
|
|
75001
76856
|
readdirSync as readdirSync8,
|
|
75002
76857
|
unlinkSync as unlinkSync5,
|
|
75003
76858
|
writeFileSync as writeFileSync9
|
|
75004
76859
|
} from "fs";
|
|
75005
76860
|
import { parseDocument as parseDocument7 } from "yaml";
|
|
75006
76861
|
function resolveOutputDirForConfig(configPath) {
|
|
75007
|
-
const base = dirname15(
|
|
76862
|
+
const base = dirname15(resolve13(configPath));
|
|
75008
76863
|
for (const dir of ["context", ".", "generated"]) {
|
|
75009
|
-
const abs =
|
|
75010
|
-
if (
|
|
76864
|
+
const abs = resolve13(base, dir);
|
|
76865
|
+
if (existsSync28(join31(abs, ".lattice", "manifest.json"))) return abs;
|
|
75011
76866
|
}
|
|
75012
|
-
return
|
|
76867
|
+
return resolve13(base, "context");
|
|
75013
76868
|
}
|
|
75014
76869
|
function friendlyConfigName(parsedName, configPath) {
|
|
75015
76870
|
if (parsedName && parsedName.trim().length > 0) return parsedName.trim();
|
|
75016
|
-
return
|
|
76871
|
+
return basename13(configPath).replace(/\.(ya?ml)$/, "");
|
|
75017
76872
|
}
|
|
75018
76873
|
function listConfigs(activeConfigPath) {
|
|
75019
76874
|
const dir = dirname15(activeConfigPath);
|
|
75020
76875
|
const entries = [];
|
|
75021
76876
|
for (const fname of readdirSync8(dir)) {
|
|
75022
76877
|
if (!fname.endsWith(".yml") && !fname.endsWith(".yaml")) continue;
|
|
75023
|
-
const full =
|
|
76878
|
+
const full = join31(dir, fname);
|
|
75024
76879
|
try {
|
|
75025
76880
|
const parsed = parseConfigFile(full);
|
|
75026
76881
|
entries.push({
|
|
@@ -75031,7 +76886,7 @@ function listConfigs(activeConfigPath) {
|
|
|
75031
76886
|
// `label` is the friendly DB name — what the user sees in the
|
|
75032
76887
|
// dropdown + settings. Falls back to the basename when unset.
|
|
75033
76888
|
label: friendlyConfigName(parsed.name, full),
|
|
75034
|
-
dbFile:
|
|
76889
|
+
dbFile: basename13(parsed.dbPath),
|
|
75035
76890
|
active: full === activeConfigPath,
|
|
75036
76891
|
// `${LATTICE_DB:...}` and postgres:// configs resolve to a
|
|
75037
76892
|
// postgres URL; everything else is a local SQLite file. This
|
|
@@ -75048,37 +76903,37 @@ function createBlankConfig(activeConfigPath, dbName) {
|
|
|
75048
76903
|
const dir = dirname15(activeConfigPath);
|
|
75049
76904
|
const slug = dbName.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
75050
76905
|
if (!slug) throw new Error("Workspace name must contain at least one alphanumeric character");
|
|
75051
|
-
const configPath =
|
|
75052
|
-
if (
|
|
76906
|
+
const configPath = join31(dir, `${slug}.config.yml`);
|
|
76907
|
+
if (existsSync28(configPath)) throw new Error(`Config already exists: ${slug}.config.yml`);
|
|
75053
76908
|
const yaml = `db: ./data/${slug}.db
|
|
75054
76909
|
|
|
75055
76910
|
entities: {}
|
|
75056
76911
|
`;
|
|
75057
76912
|
writeFileSync9(configPath, yaml, "utf8");
|
|
75058
|
-
mkdirSync11(
|
|
76913
|
+
mkdirSync11(join31(dir, "data"), { recursive: true });
|
|
75059
76914
|
return configPath;
|
|
75060
76915
|
}
|
|
75061
76916
|
function sqliteFileForConfig(configPath) {
|
|
75062
|
-
const dbVal = parseDocument7(
|
|
76917
|
+
const dbVal = parseDocument7(readFileSync25(configPath, "utf8")).get("db");
|
|
75063
76918
|
const raw = (typeof dbVal === "string" ? dbVal : "").trim();
|
|
75064
76919
|
if (!raw) return null;
|
|
75065
76920
|
if (isPostgresUrl(raw) || raw.startsWith("${LATTICE_DB:")) return null;
|
|
75066
76921
|
if (raw === ":memory:" || raw.startsWith("file:")) return null;
|
|
75067
|
-
return
|
|
76922
|
+
return resolve13(dirname15(configPath), raw);
|
|
75068
76923
|
}
|
|
75069
76924
|
function deleteDatabaseFiles(targetConfigPath) {
|
|
75070
76925
|
const sqliteFile = sqliteFileForConfig(targetConfigPath);
|
|
75071
76926
|
unlinkSync5(targetConfigPath);
|
|
75072
76927
|
let deletedDbFile = null;
|
|
75073
|
-
if (sqliteFile &&
|
|
76928
|
+
if (sqliteFile && existsSync28(sqliteFile)) {
|
|
75074
76929
|
unlinkSync5(sqliteFile);
|
|
75075
76930
|
deletedDbFile = sqliteFile;
|
|
75076
76931
|
for (const suffix of ["-wal", "-shm", "-journal"]) {
|
|
75077
76932
|
const sidecar = sqliteFile + suffix;
|
|
75078
|
-
if (
|
|
76933
|
+
if (existsSync28(sidecar)) unlinkSync5(sidecar);
|
|
75079
76934
|
}
|
|
75080
76935
|
}
|
|
75081
|
-
return { deletedConfig:
|
|
76936
|
+
return { deletedConfig: basename13(targetConfigPath), deletedDbFile };
|
|
75082
76937
|
}
|
|
75083
76938
|
|
|
75084
76939
|
// src/gui/databases-routes.ts
|
|
@@ -75094,7 +76949,7 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
75094
76949
|
sendJson(res, {
|
|
75095
76950
|
current: {
|
|
75096
76951
|
path: active.configPath,
|
|
75097
|
-
dbFile:
|
|
76952
|
+
dbFile: basename14(parsedActive.dbPath),
|
|
75098
76953
|
label: friendlyLabel,
|
|
75099
76954
|
kind
|
|
75100
76955
|
},
|
|
@@ -75108,8 +76963,8 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
75108
76963
|
sendJson(res, { error: "path must be a string" }, 400);
|
|
75109
76964
|
return true;
|
|
75110
76965
|
}
|
|
75111
|
-
const newPath =
|
|
75112
|
-
if (!
|
|
76966
|
+
const newPath = resolve14(body.path);
|
|
76967
|
+
if (!existsSync29(newPath)) {
|
|
75113
76968
|
sendJson(res, { error: `Config not found: ${newPath}` }, 400);
|
|
75114
76969
|
return true;
|
|
75115
76970
|
}
|
|
@@ -75151,16 +77006,16 @@ async function handleDatabasesRoutes(req, res, ctx, deps) {
|
|
|
75151
77006
|
sendJson(res, { error: "path must be a non-empty string" }, 400);
|
|
75152
77007
|
return true;
|
|
75153
77008
|
}
|
|
75154
|
-
const target =
|
|
77009
|
+
const target = resolve14(body.path);
|
|
75155
77010
|
const known = listConfigs(active.configPath);
|
|
75156
|
-
const match = known.find((c6) =>
|
|
77011
|
+
const match = known.find((c6) => resolve14(c6.path) === target);
|
|
75157
77012
|
if (!match) {
|
|
75158
77013
|
sendJson(res, { error: `Not a known database config: ${target}` }, 400);
|
|
75159
77014
|
return true;
|
|
75160
77015
|
}
|
|
75161
77016
|
let switchedTo = null;
|
|
75162
|
-
if (
|
|
75163
|
-
const fallback = known.find((c6) =>
|
|
77017
|
+
if (resolve14(active.configPath) === target) {
|
|
77018
|
+
const fallback = known.find((c6) => resolve14(c6.path) !== target);
|
|
75164
77019
|
if (!fallback) {
|
|
75165
77020
|
sendJson(
|
|
75166
77021
|
res,
|
|
@@ -75251,20 +77106,26 @@ async function listenWithPortFallback(server, startPort, host) {
|
|
|
75251
77106
|
throw new Error(`No available port found starting at ${String(startPort)}`);
|
|
75252
77107
|
}
|
|
75253
77108
|
async function startGuiServer(options) {
|
|
75254
|
-
const bootConfigPath = options.configPath ?
|
|
75255
|
-
const bootOutputDir = options.outputDir ?
|
|
77109
|
+
const bootConfigPath = options.configPath ? resolve15(options.configPath) : null;
|
|
77110
|
+
const bootOutputDir = options.outputDir ? resolve15(options.outputDir) : null;
|
|
75256
77111
|
const startPort = options.port ?? 4317;
|
|
75257
77112
|
const host = options.host ?? "127.0.0.1";
|
|
77113
|
+
const isLoopbackHost2 = host === "localhost" || host === "::1" || host.startsWith("127.");
|
|
77114
|
+
if (!isLoopbackHost2) {
|
|
77115
|
+
console.warn(
|
|
77116
|
+
`[lattice] GUI is binding to a non-loopback address (${host}); its data routes are UNAUTHENTICATED and will be reachable from the network.`
|
|
77117
|
+
);
|
|
77118
|
+
}
|
|
75258
77119
|
const autoRender = options.autoRender ?? false;
|
|
75259
77120
|
const guiVersion = options.version ?? "";
|
|
75260
77121
|
const sessionId = crypto.randomUUID();
|
|
75261
77122
|
let updateService = null;
|
|
75262
77123
|
let activeRef = bootConfigPath && bootOutputDir ? await openConfig(bootConfigPath, bootOutputDir, autoRender, options.realtimeWatchdogMs) : null;
|
|
75263
|
-
const latticeRoot = (bootConfigPath ? findLatticeRoot(dirname16(bootConfigPath)) : null) ?? (options.latticeRoot ?
|
|
77124
|
+
const latticeRoot = (bootConfigPath ? findLatticeRoot(dirname16(bootConfigPath)) : null) ?? (options.latticeRoot ? resolve15(options.latticeRoot) : null);
|
|
75264
77125
|
let currentWorkspaceId = null;
|
|
75265
77126
|
if (latticeRoot && bootConfigPath) {
|
|
75266
77127
|
const launched = listWorkspaces(latticeRoot).find(
|
|
75267
|
-
(w2) =>
|
|
77128
|
+
(w2) => resolve15(resolveWorkspacePaths(latticeRoot, w2).configPath) === resolve15(bootConfigPath)
|
|
75268
77129
|
);
|
|
75269
77130
|
if (launched) {
|
|
75270
77131
|
currentWorkspaceId = launched.id;
|
|
@@ -75609,6 +77470,22 @@ async function startGuiServer(options) {
|
|
|
75609
77470
|
});
|
|
75610
77471
|
}
|
|
75611
77472
|
},
|
|
77473
|
+
// ── Structured-source import (apply) ──
|
|
77474
|
+
// The importer is reachable only via dropping a file in the assistant
|
|
77475
|
+
// chat; this materializes the user-confirmed proposal, re-reading the
|
|
77476
|
+
// file's bytes from its `fileId` (its retained blob).
|
|
77477
|
+
{
|
|
77478
|
+
handle: async (req2, res2) => {
|
|
77479
|
+
if (!pathname.startsWith("/api/import/")) return false;
|
|
77480
|
+
return await dispatchImportRoute(req2, res2, {
|
|
77481
|
+
db: active.db,
|
|
77482
|
+
configPath: active.configPath,
|
|
77483
|
+
latticeRoot: dirname16(active.configPath),
|
|
77484
|
+
validTables: active.validTables,
|
|
77485
|
+
softDeletable: active.softDeletable
|
|
77486
|
+
});
|
|
77487
|
+
}
|
|
77488
|
+
},
|
|
75612
77489
|
// ── Files: blob serving + open-in-finder ──
|
|
75613
77490
|
{
|
|
75614
77491
|
handle: async (req2, res2) => {
|
|
@@ -75799,6 +77676,7 @@ ${e6.stack ?? ""}`
|
|
|
75799
77676
|
server,
|
|
75800
77677
|
port,
|
|
75801
77678
|
url,
|
|
77679
|
+
whenConverged: () => activeRef?.converged ?? Promise.resolve(),
|
|
75802
77680
|
close: () => new Promise((resolveClose, reject) => {
|
|
75803
77681
|
updateService?.stop();
|
|
75804
77682
|
for (const client of wss.clients) {
|
|
@@ -75825,8 +77703,8 @@ ${e6.stack ?? ""}`
|
|
|
75825
77703
|
}
|
|
75826
77704
|
|
|
75827
77705
|
// src/cloud/file-source-key-store.ts
|
|
75828
|
-
import { readFileSync as
|
|
75829
|
-
import { dirname as dirname17, resolve as
|
|
77706
|
+
import { readFileSync as readFileSync26, writeFileSync as writeFileSync10, existsSync as existsSync30, mkdirSync as mkdirSync12, renameSync as renameSync5, chmodSync as chmodSync3 } from "fs";
|
|
77707
|
+
import { dirname as dirname17, resolve as resolve16 } from "path";
|
|
75830
77708
|
import { createCipheriv as createCipheriv3, createDecipheriv as createDecipheriv3, randomBytes as randomBytes9, scryptSync as scryptSync3 } from "crypto";
|
|
75831
77709
|
var ENC_HEADER = "LATTICE-KMS-v1\n";
|
|
75832
77710
|
var SCRYPT_N = 1 << 15;
|
|
@@ -75844,7 +77722,7 @@ var FileSourceKeyStore = class {
|
|
|
75844
77722
|
if (!opts.path || typeof opts.path !== "string") {
|
|
75845
77723
|
throw new Error("lattice: FileSourceKeyStore requires a non-empty `path`");
|
|
75846
77724
|
}
|
|
75847
|
-
this.path =
|
|
77725
|
+
this.path = resolve16(opts.path);
|
|
75848
77726
|
this.passphrase = opts.passphrase;
|
|
75849
77727
|
this.cache = this.load();
|
|
75850
77728
|
}
|
|
@@ -75877,12 +77755,12 @@ var FileSourceKeyStore = class {
|
|
|
75877
77755
|
// ── internals ────────────────────────────────────────────────────────
|
|
75878
77756
|
load() {
|
|
75879
77757
|
const out = /* @__PURE__ */ new Map();
|
|
75880
|
-
if (!
|
|
77758
|
+
if (!existsSync30(this.path)) {
|
|
75881
77759
|
const dir = dirname17(this.path);
|
|
75882
|
-
if (!
|
|
77760
|
+
if (!existsSync30(dir)) mkdirSync12(dir, { recursive: true, mode: 448 });
|
|
75883
77761
|
return out;
|
|
75884
77762
|
}
|
|
75885
|
-
const raw =
|
|
77763
|
+
const raw = readFileSync26(this.path);
|
|
75886
77764
|
const json = this.decodeFile(raw);
|
|
75887
77765
|
for (const [sourceId, b64] of Object.entries(json)) {
|
|
75888
77766
|
try {
|
|
@@ -75991,6 +77869,7 @@ export {
|
|
|
75991
77869
|
DEFAULT_TYPE_ALIASES,
|
|
75992
77870
|
EMBEDDINGS_TABLE,
|
|
75993
77871
|
EmbeddingDimensionMismatchError,
|
|
77872
|
+
EmbeddingScanTooLargeError,
|
|
75994
77873
|
FileSourceKeyStore,
|
|
75995
77874
|
FoldCache,
|
|
75996
77875
|
InMemorySourceKeyStore,
|
|
@@ -76054,6 +77933,7 @@ export {
|
|
|
76054
77933
|
createS3Store,
|
|
76055
77934
|
createSQLiteStateStore,
|
|
76056
77935
|
decrypt,
|
|
77936
|
+
dedupeAndDetectViews,
|
|
76057
77937
|
defaultWorkspaceYaml,
|
|
76058
77938
|
deleteDbCredential,
|
|
76059
77939
|
deleteToken,
|
|
@@ -76061,6 +77941,9 @@ export {
|
|
|
76061
77941
|
deriveKey,
|
|
76062
77942
|
describeImage,
|
|
76063
77943
|
describePdf,
|
|
77944
|
+
detectAsOf,
|
|
77945
|
+
detectAsOfCandidates,
|
|
77946
|
+
detectAsOfColumns,
|
|
76064
77947
|
detectRetrievalRegressions,
|
|
76065
77948
|
diagnoseRetrieval,
|
|
76066
77949
|
discoverCloudTables,
|
|
@@ -76078,6 +77961,7 @@ export {
|
|
|
76078
77961
|
entityFileNames,
|
|
76079
77962
|
estimateTokens,
|
|
76080
77963
|
evaluateRetrieval,
|
|
77964
|
+
excelToRecords,
|
|
76081
77965
|
extractEdgesFromColumn,
|
|
76082
77966
|
extractObjects,
|
|
76083
77967
|
filePresignSql,
|
|
@@ -76106,6 +77990,8 @@ export {
|
|
|
76106
77990
|
hashFile,
|
|
76107
77991
|
hybridSearch,
|
|
76108
77992
|
importLegacyUserConfig,
|
|
77993
|
+
inferFieldType,
|
|
77994
|
+
inferSchema,
|
|
76109
77995
|
installCloudRls,
|
|
76110
77996
|
installCloudSettings,
|
|
76111
77997
|
installFilePresigner,
|
|
@@ -76124,15 +78010,19 @@ export {
|
|
|
76124
78010
|
loadColumnPolicy,
|
|
76125
78011
|
manifestPath,
|
|
76126
78012
|
markdownTable,
|
|
78013
|
+
matchSchemaToExisting,
|
|
78014
|
+
materializeImport,
|
|
76127
78015
|
memberGroupFor,
|
|
76128
78016
|
memberRoleName,
|
|
76129
78017
|
migrateLatticeData,
|
|
76130
78018
|
neighbors,
|
|
78019
|
+
normalizeName,
|
|
76131
78020
|
observationVisible,
|
|
76132
78021
|
observationsFromChange,
|
|
76133
78022
|
openTargetLatticeForMigration,
|
|
76134
78023
|
openUnderSource,
|
|
76135
78024
|
organizeSource,
|
|
78025
|
+
parseCellDate,
|
|
76136
78026
|
parseConfigFile,
|
|
76137
78027
|
parseConfigString,
|
|
76138
78028
|
parseMarkdownEntries,
|
|
@@ -76160,6 +78050,7 @@ export {
|
|
|
76160
78050
|
registryPath,
|
|
76161
78051
|
removeEdge,
|
|
76162
78052
|
removeEmbedding,
|
|
78053
|
+
renameEntities,
|
|
76163
78054
|
resolveActiveS3Config,
|
|
76164
78055
|
resolveLatticeRoot,
|
|
76165
78056
|
resolveProvenanceFields,
|
|
@@ -76190,6 +78081,7 @@ export {
|
|
|
76190
78081
|
setTableNeverShare,
|
|
76191
78082
|
shredSource,
|
|
76192
78083
|
slugify,
|
|
78084
|
+
sourceRecords,
|
|
76193
78085
|
startGuiServer,
|
|
76194
78086
|
storeEmbedding,
|
|
76195
78087
|
summarizeText,
|