@fortemi/core 2026.7.4 → 2026.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aiwg-index-schema.d.ts +9 -0
- package/dist/aiwg-index-schema.js +724 -0
- package/dist/aiwg-index-schema.js.map +1 -0
- package/dist/aiwg-index.d.ts +514 -1
- package/dist/aiwg-index.js +22 -2374
- package/dist/aiwg-index.js.map +1 -1
- package/dist/index.d.ts +1516 -18
- package/dist/index.js +2939 -1430
- package/dist/index.js.map +1 -1
- package/package.json +8 -2
- package/dist/aiwg-index-C2G7iy-b.d.ts +0 -826
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { v7 } from 'uuid';
|
|
1
|
+
import { v7, v5 } from 'uuid';
|
|
2
2
|
import { sha256 } from '@noble/hashes/sha256';
|
|
3
3
|
import { blake3 } from '@noble/hashes/blake3';
|
|
4
4
|
import { bytesToHex } from '@noble/hashes/utils';
|
|
@@ -743,6 +743,80 @@ var migration0016 = {
|
|
|
743
743
|
`
|
|
744
744
|
};
|
|
745
745
|
|
|
746
|
+
// src/migrations/0017_attachment_blob_parity.ts
|
|
747
|
+
var migration0017 = {
|
|
748
|
+
version: 17,
|
|
749
|
+
name: "0017_attachment_blob_parity",
|
|
750
|
+
sql: `
|
|
751
|
+
-- \u2500\u2500 attachment_blob parity \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
752
|
+
ALTER TABLE attachment_blob
|
|
753
|
+
ADD COLUMN IF NOT EXISTS content_type TEXT NOT NULL DEFAULT 'application/octet-stream';
|
|
754
|
+
|
|
755
|
+
ALTER TABLE attachment_blob
|
|
756
|
+
ADD COLUMN IF NOT EXISTS storage_type TEXT NOT NULL DEFAULT 'bytecask';
|
|
757
|
+
|
|
758
|
+
-- DERIVED projection state (ADR-013): rebuilt from canonical manifests;
|
|
759
|
+
-- no trigger maintains it and nothing may treat it as lifecycle truth.
|
|
760
|
+
ALTER TABLE attachment_blob
|
|
761
|
+
ADD COLUMN IF NOT EXISTS reference_count INTEGER NOT NULL DEFAULT 0;
|
|
762
|
+
|
|
763
|
+
-- Backfill content_type from the attachment-level MIME denormalization (0010).
|
|
764
|
+
UPDATE attachment_blob ab
|
|
765
|
+
SET content_type = a.mime_type
|
|
766
|
+
FROM attachment a
|
|
767
|
+
WHERE a.blob_id = ab.id
|
|
768
|
+
AND a.mime_type IS NOT NULL
|
|
769
|
+
AND ab.content_type = 'application/octet-stream';
|
|
770
|
+
|
|
771
|
+
-- Backfill the derived reference count from live manifests.
|
|
772
|
+
UPDATE attachment_blob ab
|
|
773
|
+
SET reference_count = (
|
|
774
|
+
SELECT COUNT(*) FROM attachment a
|
|
775
|
+
WHERE a.blob_id = ab.id AND a.deleted_at IS NULL
|
|
776
|
+
);
|
|
777
|
+
|
|
778
|
+
-- Orphan scan support (projection-side reporting only).
|
|
779
|
+
CREATE INDEX IF NOT EXISTS idx_attachment_blob_orphan
|
|
780
|
+
ON attachment_blob(reference_count) WHERE reference_count = 0;
|
|
781
|
+
|
|
782
|
+
-- \u2500\u2500 attachment parity additions (additive, nullable-first) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
783
|
+
ALTER TABLE attachment ADD COLUMN IF NOT EXISTS original_filename TEXT;
|
|
784
|
+
ALTER TABLE attachment ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'uploaded'
|
|
785
|
+
CHECK (status IN ('uploaded', 'queued', 'processing', 'completed', 'failed', 'quarantined'));
|
|
786
|
+
ALTER TABLE attachment ADD COLUMN IF NOT EXISTS processing_error TEXT;
|
|
787
|
+
ALTER TABLE attachment ADD COLUMN IF NOT EXISTS extraction_strategy TEXT;
|
|
788
|
+
ALTER TABLE attachment ADD COLUMN IF NOT EXISTS extraction_config JSONB DEFAULT '{}';
|
|
789
|
+
ALTER TABLE attachment ADD COLUMN IF NOT EXISTS extracted_metadata JSONB;
|
|
790
|
+
ALTER TABLE attachment ADD COLUMN IF NOT EXISTS ai_description TEXT;
|
|
791
|
+
ALTER TABLE attachment ADD COLUMN IF NOT EXISTS has_preview BOOLEAN NOT NULL DEFAULT FALSE;
|
|
792
|
+
ALTER TABLE attachment ADD COLUMN IF NOT EXISTS preview_blob_id TEXT REFERENCES attachment_blob(id);
|
|
793
|
+
|
|
794
|
+
-- Backfill: browser attachments that already carry extracted text were
|
|
795
|
+
-- fully processed at attach time \u2014 mark them completed so the
|
|
796
|
+
-- status-gated searchable-text join (server parity) keeps including them.
|
|
797
|
+
UPDATE attachment SET status = 'completed'
|
|
798
|
+
WHERE extracted_text IS NOT NULL AND extracted_text <> '';
|
|
799
|
+
|
|
800
|
+
-- \u2500\u2500 attachment_embedding (server parity; CLIP column reserved) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
801
|
+
CREATE TABLE IF NOT EXISTS attachment_embedding (
|
|
802
|
+
id TEXT PRIMARY KEY,
|
|
803
|
+
attachment_id TEXT NOT NULL REFERENCES attachment(id),
|
|
804
|
+
embedding_set_id TEXT REFERENCES embedding_set(id),
|
|
805
|
+
chunk_index INTEGER NOT NULL DEFAULT 0,
|
|
806
|
+
text TEXT NOT NULL,
|
|
807
|
+
vector vector(768),
|
|
808
|
+
clip_vector vector(512),
|
|
809
|
+
model TEXT NOT NULL,
|
|
810
|
+
embedding_type TEXT NOT NULL DEFAULT 'text',
|
|
811
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
812
|
+
UNIQUE (attachment_id, embedding_set_id, chunk_index)
|
|
813
|
+
);
|
|
814
|
+
|
|
815
|
+
CREATE INDEX IF NOT EXISTS idx_attachment_embedding_attachment
|
|
816
|
+
ON attachment_embedding(attachment_id);
|
|
817
|
+
`
|
|
818
|
+
};
|
|
819
|
+
|
|
746
820
|
// src/migrations/index.ts
|
|
747
821
|
var allMigrations = [
|
|
748
822
|
migration0001,
|
|
@@ -760,7 +834,8 @@ var allMigrations = [
|
|
|
760
834
|
migration0013,
|
|
761
835
|
migration0014,
|
|
762
836
|
migration0015,
|
|
763
|
-
migration0016
|
|
837
|
+
migration0016,
|
|
838
|
+
migration0017
|
|
764
839
|
];
|
|
765
840
|
|
|
766
841
|
// src/data-archive.ts
|
|
@@ -888,7 +963,7 @@ var NotesRepository = class {
|
|
|
888
963
|
* All writes happen in a single transaction.
|
|
889
964
|
*/
|
|
890
965
|
async create(input) {
|
|
891
|
-
const noteId = generateId();
|
|
966
|
+
const noteId = input.id ?? generateId();
|
|
892
967
|
const originalId = generateId();
|
|
893
968
|
const contentHash = computeHash(new TextEncoder().encode(input.content));
|
|
894
969
|
await this.db.transaction(async (tx) => {
|
|
@@ -1282,7 +1357,7 @@ var ATTACHMENT_TEXT_JOIN = `
|
|
|
1282
1357
|
string_agg(extracted_text, ' ' ORDER BY position, created_at)
|
|
1283
1358
|
FILTER (WHERE extracted_text IS NOT NULL AND extracted_text <> '') as extracted_text
|
|
1284
1359
|
FROM attachment
|
|
1285
|
-
WHERE deleted_at IS NULL
|
|
1360
|
+
WHERE deleted_at IS NULL AND status = 'completed'
|
|
1286
1361
|
GROUP BY note_id
|
|
1287
1362
|
) ax ON ax.note_id = n.id`;
|
|
1288
1363
|
var COMBINED_TEXT_VECTOR_SQL = `to_tsvector('english', (coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, '')))`;
|
|
@@ -1797,7 +1872,7 @@ var ATTACHMENT_TEXT_JOIN2 = `
|
|
|
1797
1872
|
string_agg(extracted_text, ' ' ORDER BY position, created_at)
|
|
1798
1873
|
FILTER (WHERE extracted_text IS NOT NULL AND extracted_text <> '') as extracted_text
|
|
1799
1874
|
FROM attachment
|
|
1800
|
-
WHERE deleted_at IS NULL
|
|
1875
|
+
WHERE deleted_at IS NULL AND status = 'completed'
|
|
1801
1876
|
GROUP BY note_id
|
|
1802
1877
|
) ax ON ax.note_id = n.id`;
|
|
1803
1878
|
var COMBINED_TEXT_SQL = `trim(both from (coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, '')))`;
|
|
@@ -3609,150 +3684,326 @@ function matchRoute(routes, request, url) {
|
|
|
3609
3684
|
return null;
|
|
3610
3685
|
}
|
|
3611
3686
|
|
|
3612
|
-
// src/blob-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3687
|
+
// src/shard/blob-sidecar.ts
|
|
3688
|
+
var SIDECAR_PREFIX = "blobs/";
|
|
3689
|
+
function blobChecksumToHex(checksum) {
|
|
3690
|
+
const sep = checksum.indexOf(":");
|
|
3691
|
+
return sep >= 0 ? checksum.slice(sep + 1) : checksum;
|
|
3692
|
+
}
|
|
3693
|
+
function sidecarEntryName(checksum) {
|
|
3694
|
+
return SIDECAR_PREFIX + blobChecksumToHex(checksum);
|
|
3695
|
+
}
|
|
3696
|
+
function isSidecarEntry(name) {
|
|
3697
|
+
if (!name.startsWith(SIDECAR_PREFIX)) return false;
|
|
3698
|
+
const rest = name.slice(SIDECAR_PREFIX.length);
|
|
3699
|
+
return rest.length > 0 && !rest.includes("/");
|
|
3619
3700
|
}
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3701
|
+
function collectSidecarBlobs(files) {
|
|
3702
|
+
const blobs = /* @__PURE__ */ new Map();
|
|
3703
|
+
for (const [name, bytes] of files) {
|
|
3704
|
+
if (isSidecarEntry(name)) {
|
|
3705
|
+
blobs.set(name.slice(SIDECAR_PREFIX.length), bytes);
|
|
3706
|
+
}
|
|
3623
3707
|
}
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3708
|
+
return blobs;
|
|
3709
|
+
}
|
|
3710
|
+
|
|
3711
|
+
// src/blob-store-legacy.ts
|
|
3712
|
+
function legacyName(archiveName) {
|
|
3713
|
+
return `fortemi-${archiveName}-blobs`;
|
|
3714
|
+
}
|
|
3715
|
+
function idbRequest(req) {
|
|
3716
|
+
return new Promise((resolve, reject) => {
|
|
3717
|
+
req.onsuccess = () => resolve(req.result);
|
|
3718
|
+
req.onerror = () => reject(req.error);
|
|
3719
|
+
});
|
|
3720
|
+
}
|
|
3721
|
+
async function legacyIdbExists(factory, name) {
|
|
3722
|
+
if (typeof factory.databases === "function") {
|
|
3723
|
+
const dbs = await factory.databases();
|
|
3724
|
+
return dbs.some((db) => db.name === name);
|
|
3627
3725
|
}
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3726
|
+
return false;
|
|
3727
|
+
}
|
|
3728
|
+
async function migrateLegacyIdb(archiveName, target, factory) {
|
|
3729
|
+
const name = legacyName(archiveName);
|
|
3730
|
+
if (!await legacyIdbExists(factory, name)) return 0;
|
|
3731
|
+
const db = await new Promise((resolve, reject) => {
|
|
3732
|
+
const req = factory.open(name);
|
|
3733
|
+
req.onsuccess = () => resolve(req.result);
|
|
3734
|
+
req.onerror = () => reject(req.error);
|
|
3735
|
+
});
|
|
3736
|
+
try {
|
|
3737
|
+
if (!db.objectStoreNames.contains("blobs")) return 0;
|
|
3738
|
+
const tx = db.transaction("blobs", "readonly");
|
|
3739
|
+
const values = await idbRequest(tx.objectStore("blobs").getAll());
|
|
3740
|
+
let migrated = 0;
|
|
3741
|
+
for (const value of values) {
|
|
3742
|
+
if (value instanceof Uint8Array) {
|
|
3743
|
+
await target.put(value);
|
|
3744
|
+
migrated += 1;
|
|
3745
|
+
} else if (value instanceof ArrayBuffer) {
|
|
3746
|
+
await target.put(new Uint8Array(value));
|
|
3747
|
+
migrated += 1;
|
|
3748
|
+
}
|
|
3749
|
+
}
|
|
3750
|
+
db.close();
|
|
3751
|
+
await idbRequest(factory.deleteDatabase(name));
|
|
3752
|
+
return migrated;
|
|
3753
|
+
} finally {
|
|
3754
|
+
db.close();
|
|
3641
3755
|
}
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
await writable.write(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength));
|
|
3647
|
-
await writable.close();
|
|
3756
|
+
}
|
|
3757
|
+
async function migrateLegacyOpfs(archiveName, target) {
|
|
3758
|
+
if (typeof navigator === "undefined" || typeof navigator.storage?.getDirectory !== "function") {
|
|
3759
|
+
return 0;
|
|
3648
3760
|
}
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3761
|
+
const root = await navigator.storage.getDirectory();
|
|
3762
|
+
let legacyDir;
|
|
3763
|
+
try {
|
|
3764
|
+
legacyDir = await root.getDirectoryHandle(legacyName(archiveName), {
|
|
3765
|
+
create: false
|
|
3766
|
+
});
|
|
3767
|
+
} catch {
|
|
3768
|
+
return 0;
|
|
3655
3769
|
}
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
const
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3770
|
+
let migrated = 0;
|
|
3771
|
+
for await (const [, d1] of legacyDir.entries()) {
|
|
3772
|
+
if (d1.kind !== "directory") continue;
|
|
3773
|
+
for await (const [, d2] of d1.entries()) {
|
|
3774
|
+
if (d2.kind !== "directory") continue;
|
|
3775
|
+
for await (const [, fh] of d2.entries()) {
|
|
3776
|
+
if (fh.kind !== "file") continue;
|
|
3777
|
+
const file = await fh.getFile();
|
|
3778
|
+
await target.put(new Uint8Array(await file.arrayBuffer()));
|
|
3779
|
+
migrated += 1;
|
|
3666
3780
|
}
|
|
3667
|
-
throw err;
|
|
3668
3781
|
}
|
|
3669
3782
|
}
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
return fh !== null;
|
|
3673
|
-
}
|
|
3674
|
-
};
|
|
3675
|
-
var IDB_STORE = "blobs";
|
|
3676
|
-
var IDB_VERSION = 1;
|
|
3677
|
-
function openDb(dbName) {
|
|
3678
|
-
return new Promise((resolve, reject) => {
|
|
3679
|
-
const req = indexedDB.open(dbName, IDB_VERSION);
|
|
3680
|
-
req.onupgradeneeded = () => {
|
|
3681
|
-
req.result.createObjectStore(IDB_STORE);
|
|
3682
|
-
};
|
|
3683
|
-
req.onsuccess = () => resolve(req.result);
|
|
3684
|
-
req.onerror = () => reject(req.error);
|
|
3685
|
-
});
|
|
3783
|
+
await root.removeEntry(legacyName(archiveName), { recursive: true });
|
|
3784
|
+
return migrated;
|
|
3686
3785
|
}
|
|
3687
|
-
|
|
3688
|
-
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
3786
|
+
async function migrateLegacyBlobStore(archiveName, target, indexedDbFactory) {
|
|
3787
|
+
let migrated = 0;
|
|
3788
|
+
let failed = false;
|
|
3789
|
+
const factory = indexedDbFactory ?? globalThis.indexedDB;
|
|
3790
|
+
if (factory) {
|
|
3791
|
+
try {
|
|
3792
|
+
migrated += await migrateLegacyIdb(archiveName, target, factory);
|
|
3793
|
+
} catch {
|
|
3794
|
+
failed = true;
|
|
3696
3795
|
}
|
|
3697
|
-
return this._db;
|
|
3698
3796
|
}
|
|
3699
|
-
|
|
3700
|
-
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
tx.objectStore(IDB_STORE).put(data, hash);
|
|
3704
|
-
tx.oncomplete = () => resolve();
|
|
3705
|
-
tx.onerror = () => reject(tx.error);
|
|
3706
|
-
});
|
|
3797
|
+
try {
|
|
3798
|
+
migrated += await migrateLegacyOpfs(archiveName, target);
|
|
3799
|
+
} catch {
|
|
3800
|
+
failed = true;
|
|
3707
3801
|
}
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
3802
|
+
return { migrated, failed };
|
|
3803
|
+
}
|
|
3804
|
+
|
|
3805
|
+
// src/blob-store.ts
|
|
3806
|
+
var MemoryBlobStore = class {
|
|
3807
|
+
constructor(now = Date.now) {
|
|
3808
|
+
this.now = now;
|
|
3809
|
+
}
|
|
3810
|
+
entries = /* @__PURE__ */ new Map();
|
|
3811
|
+
async put(bytes) {
|
|
3812
|
+
const checksum = computeBlobHash(bytes);
|
|
3813
|
+
const existing = this.entries.get(checksum);
|
|
3814
|
+
if (existing) {
|
|
3815
|
+
existing.refcount += 1;
|
|
3816
|
+
} else {
|
|
3817
|
+
this.entries.set(checksum, { bytes, createdAt: this.now(), refcount: 1 });
|
|
3818
|
+
}
|
|
3819
|
+
return checksum;
|
|
3820
|
+
}
|
|
3821
|
+
async read(checksum) {
|
|
3822
|
+
return this.entries.get(checksum)?.bytes ?? null;
|
|
3823
|
+
}
|
|
3824
|
+
async has(checksum) {
|
|
3825
|
+
return this.entries.has(checksum);
|
|
3826
|
+
}
|
|
3827
|
+
async reconcile(liveChecksums, opts) {
|
|
3828
|
+
const live = /* @__PURE__ */ new Map();
|
|
3829
|
+
for (const checksum of liveChecksums) {
|
|
3830
|
+
live.set(checksum, (live.get(checksum) ?? 0) + 1);
|
|
3831
|
+
}
|
|
3832
|
+
const missing = [...live.keys()].filter((c) => !this.entries.has(c)).sort();
|
|
3833
|
+
const unreferenced = [];
|
|
3834
|
+
let referenced = 0;
|
|
3835
|
+
let removed = 0;
|
|
3836
|
+
let bytesFreed = 0;
|
|
3837
|
+
for (const [checksum, entry] of [...this.entries].sort()) {
|
|
3838
|
+
const refcount = live.get(checksum) ?? 0;
|
|
3839
|
+
entry.refcount = refcount;
|
|
3840
|
+
if (refcount > 0) {
|
|
3841
|
+
referenced += 1;
|
|
3842
|
+
continue;
|
|
3843
|
+
}
|
|
3844
|
+
unreferenced.push(checksum);
|
|
3845
|
+
if (opts?.removeUnreferenced) {
|
|
3846
|
+
this.entries.delete(checksum);
|
|
3847
|
+
removed += 1;
|
|
3848
|
+
bytesFreed += entry.bytes.byteLength;
|
|
3849
|
+
}
|
|
3850
|
+
}
|
|
3851
|
+
return { referenced, missing, unreferenced, removed, bytesFreed };
|
|
3716
3852
|
}
|
|
3717
|
-
async
|
|
3718
|
-
const
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
3724
|
-
|
|
3853
|
+
async gc(opts) {
|
|
3854
|
+
const minAgeMs = opts?.minAgeMs ?? 0;
|
|
3855
|
+
const cutoff = this.now() - minAgeMs;
|
|
3856
|
+
let collected = 0;
|
|
3857
|
+
let bytesFreed = 0;
|
|
3858
|
+
for (const [checksum, entry] of this.entries) {
|
|
3859
|
+
if (entry.refcount === 0 && entry.createdAt <= cutoff) {
|
|
3860
|
+
this.entries.delete(checksum);
|
|
3861
|
+
collected += 1;
|
|
3862
|
+
bytesFreed += entry.bytes.byteLength;
|
|
3863
|
+
}
|
|
3864
|
+
}
|
|
3865
|
+
return { collected, bytesFreed };
|
|
3725
3866
|
}
|
|
3726
|
-
async
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
|
|
3730
|
-
const req = tx.objectStore(IDB_STORE).count(hash);
|
|
3731
|
-
req.onsuccess = () => resolve(req.result > 0);
|
|
3732
|
-
req.onerror = () => reject(req.error);
|
|
3733
|
-
});
|
|
3867
|
+
async diagnostics() {
|
|
3868
|
+
return { backend: "memory", probe: null };
|
|
3869
|
+
}
|
|
3870
|
+
async close() {
|
|
3734
3871
|
}
|
|
3735
3872
|
};
|
|
3736
|
-
var
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
|
|
3873
|
+
var CHECKSUM_PREFIX = "blake3:";
|
|
3874
|
+
function toChecksum(hex) {
|
|
3875
|
+
return CHECKSUM_PREFIX + hex;
|
|
3876
|
+
}
|
|
3877
|
+
var BytecaskBlobStore = class {
|
|
3878
|
+
constructor(facade, adapter, index, probe, now = Date.now) {
|
|
3879
|
+
this.facade = facade;
|
|
3880
|
+
this.adapter = adapter;
|
|
3881
|
+
this.index = index;
|
|
3882
|
+
this.probe = probe;
|
|
3883
|
+
this.now = now;
|
|
3884
|
+
}
|
|
3885
|
+
async put(bytes) {
|
|
3886
|
+
return toChecksum(await this.facade.put(bytes));
|
|
3887
|
+
}
|
|
3888
|
+
async read(checksum) {
|
|
3889
|
+
return this.facade.get(blobChecksumToHex(checksum));
|
|
3890
|
+
}
|
|
3891
|
+
async has(checksum) {
|
|
3892
|
+
return this.adapter.exists(blobChecksumToHex(checksum));
|
|
3893
|
+
}
|
|
3894
|
+
async reconcile(liveChecksums, opts) {
|
|
3895
|
+
const live = /* @__PURE__ */ new Set();
|
|
3896
|
+
for (const checksum of liveChecksums) live.add(blobChecksumToHex(checksum));
|
|
3897
|
+
const physical = /* @__PURE__ */ new Set();
|
|
3898
|
+
for await (const hash of this.adapter.list()) physical.add(hash);
|
|
3899
|
+
const entryList = [...await this.index.values()];
|
|
3900
|
+
const entries = new Map(entryList.map((e) => [e.hash, e]));
|
|
3901
|
+
for (const hash of [...entries.keys()]) {
|
|
3902
|
+
if (!physical.has(hash)) {
|
|
3903
|
+
await this.index.delete(hash);
|
|
3904
|
+
entries.delete(hash);
|
|
3905
|
+
}
|
|
3906
|
+
}
|
|
3907
|
+
const missing = [...live].filter((hash) => !physical.has(hash)).sort().map(toChecksum);
|
|
3908
|
+
const unreferenced = [];
|
|
3909
|
+
let referenced = 0;
|
|
3910
|
+
let removed = 0;
|
|
3911
|
+
let bytesFreed = 0;
|
|
3912
|
+
for (const hash of [...physical].sort()) {
|
|
3913
|
+
const existing = entries.get(hash);
|
|
3914
|
+
const size = existing?.size ?? (await this.adapter.read(hash))?.byteLength ?? 0;
|
|
3915
|
+
if (live.has(hash)) {
|
|
3916
|
+
await this.index.set({
|
|
3917
|
+
hash,
|
|
3918
|
+
size,
|
|
3919
|
+
createdAt: existing?.createdAt ?? this.now(),
|
|
3920
|
+
refcount: 1
|
|
3921
|
+
});
|
|
3922
|
+
referenced += 1;
|
|
3923
|
+
continue;
|
|
3924
|
+
}
|
|
3925
|
+
unreferenced.push(toChecksum(hash));
|
|
3926
|
+
if (opts?.removeUnreferenced) {
|
|
3927
|
+
await this.adapter.remove(hash);
|
|
3928
|
+
await this.index.delete(hash);
|
|
3929
|
+
removed += 1;
|
|
3930
|
+
bytesFreed += size;
|
|
3931
|
+
} else {
|
|
3932
|
+
await this.index.set({
|
|
3933
|
+
hash,
|
|
3934
|
+
size,
|
|
3935
|
+
createdAt: existing?.createdAt ?? this.now(),
|
|
3936
|
+
refcount: 0
|
|
3937
|
+
});
|
|
3938
|
+
}
|
|
3939
|
+
}
|
|
3940
|
+
return { referenced, missing, unreferenced, removed, bytesFreed };
|
|
3740
3941
|
}
|
|
3741
|
-
async
|
|
3742
|
-
|
|
3942
|
+
async gc(opts) {
|
|
3943
|
+
const result = await this.facade.gc({ minAgeMs: opts?.minAgeMs });
|
|
3944
|
+
return { collected: result.collected, bytesFreed: result.bytesFreed };
|
|
3743
3945
|
}
|
|
3744
|
-
async
|
|
3745
|
-
this.
|
|
3946
|
+
async diagnostics() {
|
|
3947
|
+
return { backend: this.adapter.kind, probe: this.probe };
|
|
3746
3948
|
}
|
|
3747
|
-
async
|
|
3748
|
-
|
|
3949
|
+
async close() {
|
|
3950
|
+
await this.facade.close();
|
|
3749
3951
|
}
|
|
3750
3952
|
};
|
|
3751
|
-
function createBlobStore(archiveName) {
|
|
3752
|
-
|
|
3753
|
-
|
|
3953
|
+
async function createBlobStore(archiveName, options) {
|
|
3954
|
+
const bytecask = await import('@bytecask/core');
|
|
3955
|
+
if (options?.backend === "opfs") {
|
|
3956
|
+
throw new Error(
|
|
3957
|
+
"BlobStore: the OPFS tier requires @bytecask/core >= 2026.7.2 (worker-backed adapter); use the default IndexedDB tier until the upstream factory switch lands."
|
|
3958
|
+
);
|
|
3959
|
+
}
|
|
3960
|
+
const idbFactory = options?.indexedDB ?? globalThis.indexedDB;
|
|
3961
|
+
const attempted = [];
|
|
3962
|
+
let adapter;
|
|
3963
|
+
const wantsIdb = options?.backend !== "memory";
|
|
3964
|
+
if (wantsIdb && idbFactory) {
|
|
3965
|
+
adapter = new bytecask.IdbAdapter({
|
|
3966
|
+
databaseName: `fortemi-${archiveName}-bytecask`,
|
|
3967
|
+
factory: idbFactory
|
|
3968
|
+
});
|
|
3969
|
+
attempted.push({ backend: "idb", ok: true });
|
|
3970
|
+
} else {
|
|
3971
|
+
if (wantsIdb) {
|
|
3972
|
+
attempted.push({ backend: "idb", ok: false, reason: "indexedDB unavailable" });
|
|
3973
|
+
if (options?.backend === "idb") {
|
|
3974
|
+
throw new Error("BlobStore: IndexedDB tier requested but indexedDB is unavailable");
|
|
3975
|
+
}
|
|
3976
|
+
}
|
|
3977
|
+
adapter = new bytecask.MemoryAdapter();
|
|
3978
|
+
attempted.push({ backend: "memory", ok: true });
|
|
3754
3979
|
}
|
|
3755
|
-
|
|
3980
|
+
const index = new bytecask.MemoryIndexStore();
|
|
3981
|
+
const facade = bytecask.createMemoryBlobStore({ adapter, index });
|
|
3982
|
+
const probe = { backend: adapter.kind, attempted };
|
|
3983
|
+
const wrapped = new BytecaskBlobStore(facade, adapter, index, probe);
|
|
3984
|
+
if (options?.migrateLegacy !== false) {
|
|
3985
|
+
await migrateLegacyBlobStore(archiveName, wrapped, options?.indexedDB);
|
|
3986
|
+
}
|
|
3987
|
+
return wrapped;
|
|
3988
|
+
}
|
|
3989
|
+
function createLazyBlobStore(archiveName, options) {
|
|
3990
|
+
let inner = null;
|
|
3991
|
+
const open = () => {
|
|
3992
|
+
inner ??= createBlobStore(archiveName, options);
|
|
3993
|
+
return inner;
|
|
3994
|
+
};
|
|
3995
|
+
return {
|
|
3996
|
+
put: async (bytes) => (await open()).put(bytes),
|
|
3997
|
+
read: async (checksum) => (await open()).read(checksum),
|
|
3998
|
+
has: async (checksum) => (await open()).has(checksum),
|
|
3999
|
+
reconcile: async (live, opts) => (await open()).reconcile(live, opts),
|
|
4000
|
+
gc: async (opts) => (await open()).gc(opts),
|
|
4001
|
+
diagnostics: async () => (await open()).diagnostics(),
|
|
4002
|
+
close: async () => {
|
|
4003
|
+
if (inner) await (await inner).close();
|
|
4004
|
+
inner = null;
|
|
4005
|
+
}
|
|
4006
|
+
};
|
|
3756
4007
|
}
|
|
3757
4008
|
|
|
3758
4009
|
// src/repositories/graph-repository.ts
|
|
@@ -5412,16 +5663,21 @@ var AttachmentsRepository = class {
|
|
|
5412
5663
|
/**
|
|
5413
5664
|
* Attach a binary file to a note.
|
|
5414
5665
|
*
|
|
5415
|
-
*
|
|
5416
|
-
*
|
|
5417
|
-
*
|
|
5418
|
-
*
|
|
5419
|
-
*
|
|
5666
|
+
* Bytes are written first (`put()` is idempotent — content addressing makes
|
|
5667
|
+
* replays safe), then the metadata rows commit (ADR-013 D5). A crash
|
|
5668
|
+
* between the two leaves an unreferenced blob that reconcile/gc sweeps —
|
|
5669
|
+
* never a manifest without recoverable state. The store-computed BLAKE3
|
|
5670
|
+
* `content_hash` (`blake3:<hex>`) matches the server convention and is the
|
|
5671
|
+
* key used by the portable Knowledge-Shard byte sidecar.
|
|
5672
|
+
*
|
|
5673
|
+
* If a blob row with the same content hash already exists, it is reused
|
|
5674
|
+
* (deduplication) — re-putting the bytes also heals a previously
|
|
5675
|
+
* reference-only blob whose bytes went missing.
|
|
5420
5676
|
*
|
|
5421
5677
|
* Returns the newly created AttachmentRow.
|
|
5422
5678
|
*/
|
|
5423
5679
|
async attach(input) {
|
|
5424
|
-
const contentHash =
|
|
5680
|
+
const contentHash = await this.blobStore.put(input.data);
|
|
5425
5681
|
const sizeBytes = input.data.length;
|
|
5426
5682
|
let blobId;
|
|
5427
5683
|
const existing = await this.db.query(
|
|
@@ -5432,7 +5688,6 @@ var AttachmentsRepository = class {
|
|
|
5432
5688
|
blobId = existing.rows[0].id;
|
|
5433
5689
|
} else {
|
|
5434
5690
|
blobId = generateId();
|
|
5435
|
-
await this.blobStore.write(contentHash, input.data);
|
|
5436
5691
|
await this.db.query(
|
|
5437
5692
|
`INSERT INTO attachment_blob (id, content_hash, size_bytes) VALUES ($1, $2, $3)`,
|
|
5438
5693
|
[blobId, contentHash, sizeBytes]
|
|
@@ -5440,8 +5695,8 @@ var AttachmentsRepository = class {
|
|
|
5440
5695
|
}
|
|
5441
5696
|
const attachmentId = generateId();
|
|
5442
5697
|
await this.db.query(
|
|
5443
|
-
`INSERT INTO attachment (id, note_id, blob_id, filename, display_name, mime_type, extracted_text)
|
|
5444
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7)`,
|
|
5698
|
+
`INSERT INTO attachment (id, note_id, blob_id, filename, display_name, mime_type, extracted_text, status)
|
|
5699
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
|
|
5445
5700
|
[
|
|
5446
5701
|
attachmentId,
|
|
5447
5702
|
input.noteId,
|
|
@@ -5449,7 +5704,8 @@ var AttachmentsRepository = class {
|
|
|
5449
5704
|
input.filename,
|
|
5450
5705
|
input.displayName ?? null,
|
|
5451
5706
|
input.mimeType ?? null,
|
|
5452
|
-
input.extractedText ?? null
|
|
5707
|
+
input.extractedText ?? null,
|
|
5708
|
+
input.extractedText ? "completed" : "uploaded"
|
|
5453
5709
|
]
|
|
5454
5710
|
);
|
|
5455
5711
|
return this.get(attachmentId);
|
|
@@ -5468,16 +5724,23 @@ var AttachmentsRepository = class {
|
|
|
5468
5724
|
}
|
|
5469
5725
|
/**
|
|
5470
5726
|
* Retrieve the raw binary data for an attachment.
|
|
5471
|
-
* Returns null
|
|
5727
|
+
* Returns null when the bytes are not present — the attachment is then in
|
|
5728
|
+
* the recoverable reference-only state (metadata intact, bytes
|
|
5729
|
+
* re-hydratable from a shard sidecar or a re-attach of the same content).
|
|
5472
5730
|
*/
|
|
5473
5731
|
async getBlob(attachmentId) {
|
|
5474
|
-
const
|
|
5475
|
-
|
|
5476
|
-
|
|
5477
|
-
|
|
5478
|
-
|
|
5479
|
-
|
|
5480
|
-
|
|
5732
|
+
const checksum = await this.blobChecksumOf(attachmentId);
|
|
5733
|
+
if (checksum === null) return null;
|
|
5734
|
+
return this.blobStore.read(checksum);
|
|
5735
|
+
}
|
|
5736
|
+
/**
|
|
5737
|
+
* True when the attachment's bytes are physically present in the BlobStore.
|
|
5738
|
+
* False means reference-only (recoverable), not an error.
|
|
5739
|
+
*/
|
|
5740
|
+
async hasBlob(attachmentId) {
|
|
5741
|
+
const checksum = await this.blobChecksumOf(attachmentId);
|
|
5742
|
+
if (checksum === null) return false;
|
|
5743
|
+
return this.blobStore.has(checksum);
|
|
5481
5744
|
}
|
|
5482
5745
|
/**
|
|
5483
5746
|
* List active (non-deleted) attachments for a note.
|
|
@@ -5494,7 +5757,9 @@ var AttachmentsRepository = class {
|
|
|
5494
5757
|
}
|
|
5495
5758
|
/**
|
|
5496
5759
|
* Soft-delete an attachment by setting deleted_at to the current timestamp.
|
|
5497
|
-
* The
|
|
5760
|
+
* The blob row is untouched and no bytes are removed inline — physical
|
|
5761
|
+
* removal happens only through deferred `reconcileBlobs()`/`gcBlobs()`
|
|
5762
|
+
* against the canonical live set (ADR-013 D4).
|
|
5498
5763
|
*/
|
|
5499
5764
|
async delete(id) {
|
|
5500
5765
|
await this.db.query(
|
|
@@ -5502,6 +5767,44 @@ var AttachmentsRepository = class {
|
|
|
5502
5767
|
[id]
|
|
5503
5768
|
);
|
|
5504
5769
|
}
|
|
5770
|
+
/**
|
|
5771
|
+
* The authoritative live-checksum set: content hashes referenced by at
|
|
5772
|
+
* least one non-deleted attachment. This — not any refcount — decides
|
|
5773
|
+
* which bytes are reachable.
|
|
5774
|
+
*/
|
|
5775
|
+
async liveBlobChecksums() {
|
|
5776
|
+
const result = await this.db.query(
|
|
5777
|
+
`SELECT DISTINCT ab.content_hash
|
|
5778
|
+
FROM attachment_blob ab
|
|
5779
|
+
JOIN attachment a ON a.blob_id = ab.id
|
|
5780
|
+
WHERE a.deleted_at IS NULL`
|
|
5781
|
+
);
|
|
5782
|
+
return result.rows.map((row) => row.content_hash);
|
|
5783
|
+
}
|
|
5784
|
+
/**
|
|
5785
|
+
* Reconcile the BlobStore against the canonical live set (startup, after
|
|
5786
|
+
* quota events, after interrupted writes). `missing` lists reference-only
|
|
5787
|
+
* checksums; `unreferenced` lists GC candidates.
|
|
5788
|
+
*/
|
|
5789
|
+
async reconcileBlobs(opts) {
|
|
5790
|
+
return this.blobStore.reconcile(await this.liveBlobChecksums(), opts);
|
|
5791
|
+
}
|
|
5792
|
+
/**
|
|
5793
|
+
* Deferred, age-thresholded physical removal of unreachable bytes.
|
|
5794
|
+
* Runs a reconcile first so GC always acts on current manifest truth.
|
|
5795
|
+
*/
|
|
5796
|
+
async gcBlobs(opts) {
|
|
5797
|
+
await this.reconcileBlobs();
|
|
5798
|
+
return this.blobStore.gc(opts);
|
|
5799
|
+
}
|
|
5800
|
+
async blobChecksumOf(attachmentId) {
|
|
5801
|
+
const att = await this.get(attachmentId);
|
|
5802
|
+
const blob = await this.db.query(
|
|
5803
|
+
`SELECT content_hash FROM attachment_blob WHERE id = $1`,
|
|
5804
|
+
[att.blob_id]
|
|
5805
|
+
);
|
|
5806
|
+
return blob.rows.length > 0 ? blob.rows[0].content_hash : null;
|
|
5807
|
+
}
|
|
5505
5808
|
};
|
|
5506
5809
|
|
|
5507
5810
|
// src/tools/manage-attachments.ts
|
|
@@ -5516,6 +5819,21 @@ var ManageAttachmentsInputSchema = z.object({
|
|
|
5516
5819
|
extracted_text: z.string().optional(),
|
|
5517
5820
|
display_name: z.string().optional()
|
|
5518
5821
|
});
|
|
5822
|
+
function toToolAttachment(row) {
|
|
5823
|
+
return {
|
|
5824
|
+
id: row.id,
|
|
5825
|
+
note_id: row.note_id,
|
|
5826
|
+
blob_id: row.blob_id,
|
|
5827
|
+
document_type_id: row.document_type_id,
|
|
5828
|
+
mime_type: row.mime_type,
|
|
5829
|
+
extracted_text: row.extracted_text,
|
|
5830
|
+
filename: row.filename,
|
|
5831
|
+
display_name: row.display_name,
|
|
5832
|
+
position: row.position,
|
|
5833
|
+
created_at: row.created_at,
|
|
5834
|
+
deleted_at: row.deleted_at
|
|
5835
|
+
};
|
|
5836
|
+
}
|
|
5519
5837
|
async function manageAttachments(db, blobStore, rawInput) {
|
|
5520
5838
|
const input = ManageAttachmentsInputSchema.parse(rawInput);
|
|
5521
5839
|
const repo = new AttachmentsRepository(db, blobStore);
|
|
@@ -5539,17 +5857,17 @@ async function manageAttachments(db, blobStore, rawInput) {
|
|
|
5539
5857
|
await enqueueJob(db, { noteId: input.note_id, jobType: "embedding" });
|
|
5540
5858
|
await enqueueJob(db, { noteId: input.note_id, jobType: "concept_tagging" });
|
|
5541
5859
|
}
|
|
5542
|
-
return { action: "attach", attachment, size_bytes: data.length };
|
|
5860
|
+
return { action: "attach", attachment: toToolAttachment(attachment), size_bytes: data.length };
|
|
5543
5861
|
}
|
|
5544
5862
|
case "list": {
|
|
5545
5863
|
if (!input.note_id) throw new Error("note_id required for list");
|
|
5546
5864
|
const attachments = await repo.list(input.note_id);
|
|
5547
|
-
return { action: "list", attachments };
|
|
5865
|
+
return { action: "list", attachments: attachments.map(toToolAttachment) };
|
|
5548
5866
|
}
|
|
5549
5867
|
case "get": {
|
|
5550
5868
|
if (!input.attachment_id) throw new Error("attachment_id required for get");
|
|
5551
5869
|
const attachment = await repo.get(input.attachment_id);
|
|
5552
|
-
return { action: "get", attachment };
|
|
5870
|
+
return { action: "get", attachment: toToolAttachment(attachment) };
|
|
5553
5871
|
}
|
|
5554
5872
|
case "get_blob": {
|
|
5555
5873
|
if (!input.attachment_id) throw new Error("attachment_id required for get_blob");
|
|
@@ -7011,6 +7329,180 @@ async function validateChecksums(checksums, files) {
|
|
|
7011
7329
|
return { valid: failures.length === 0, failures };
|
|
7012
7330
|
}
|
|
7013
7331
|
|
|
7332
|
+
// src/shard/shard-signature.ts
|
|
7333
|
+
var SIGNATURE_ENTRY = "signature.json";
|
|
7334
|
+
var SIGNING_ENVELOPE_VERSION = "1";
|
|
7335
|
+
var SIGNING_ALGORITHM = "ed25519";
|
|
7336
|
+
var MAX_ENVELOPE_BYTES = 64 * 1024;
|
|
7337
|
+
var AllowlistTrustStore = class {
|
|
7338
|
+
keys;
|
|
7339
|
+
constructor(keys) {
|
|
7340
|
+
this.keys = new Map(keys.map((k) => [k.key_id, k]));
|
|
7341
|
+
}
|
|
7342
|
+
resolve(keyId) {
|
|
7343
|
+
return this.keys.get(keyId) ?? null;
|
|
7344
|
+
}
|
|
7345
|
+
/** Mark a key revoked without removing it (still resolvable, verdict `revoked`). */
|
|
7346
|
+
revoke(keyId) {
|
|
7347
|
+
const key = this.keys.get(keyId);
|
|
7348
|
+
if (key) this.keys.set(keyId, { ...key, revoked: true });
|
|
7349
|
+
}
|
|
7350
|
+
};
|
|
7351
|
+
function base64urlToBytes(s) {
|
|
7352
|
+
const padded = s.replace(/-/g, "+").replace(/_/g, "/");
|
|
7353
|
+
const b64 = padded + "=".repeat((4 - padded.length % 4) % 4);
|
|
7354
|
+
const bin = atob(b64);
|
|
7355
|
+
const out = new Uint8Array(bin.length);
|
|
7356
|
+
for (let i = 0; i < bin.length; i += 1) out[i] = bin.charCodeAt(i);
|
|
7357
|
+
return out;
|
|
7358
|
+
}
|
|
7359
|
+
function bytesToBase64url(bytes) {
|
|
7360
|
+
let bin = "";
|
|
7361
|
+
for (const b of bytes) bin += String.fromCharCode(b);
|
|
7362
|
+
return btoa(bin).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
|
7363
|
+
}
|
|
7364
|
+
function canonicalPayloadBytes(payload) {
|
|
7365
|
+
const canonical = {
|
|
7366
|
+
blob_digests: [...payload.blob_digests].sort(),
|
|
7367
|
+
format_version: payload.format_version,
|
|
7368
|
+
manifest_digest: payload.manifest_digest,
|
|
7369
|
+
signer: {
|
|
7370
|
+
algorithm: payload.signer.algorithm,
|
|
7371
|
+
key_id: payload.signer.key_id,
|
|
7372
|
+
public_key: payload.signer.public_key
|
|
7373
|
+
}
|
|
7374
|
+
};
|
|
7375
|
+
return new TextEncoder().encode(JSON.stringify(canonical));
|
|
7376
|
+
}
|
|
7377
|
+
var ed25519Supported = null;
|
|
7378
|
+
async function isShardSigningSupported() {
|
|
7379
|
+
if (ed25519Supported !== null) return ed25519Supported;
|
|
7380
|
+
try {
|
|
7381
|
+
const subtle = globalThis.crypto?.subtle;
|
|
7382
|
+
if (!subtle) {
|
|
7383
|
+
ed25519Supported = false;
|
|
7384
|
+
return false;
|
|
7385
|
+
}
|
|
7386
|
+
await subtle.importKey("raw", toBufferSource(new Uint8Array(32)), { name: "Ed25519" }, false, ["verify"]);
|
|
7387
|
+
ed25519Supported = true;
|
|
7388
|
+
} catch {
|
|
7389
|
+
ed25519Supported = false;
|
|
7390
|
+
}
|
|
7391
|
+
return ed25519Supported;
|
|
7392
|
+
}
|
|
7393
|
+
function sidecarBlobDigests(files) {
|
|
7394
|
+
const digests = [];
|
|
7395
|
+
for (const name of files.keys()) {
|
|
7396
|
+
if (isSidecarEntry(name)) digests.push(name.slice(SIDECAR_PREFIX.length));
|
|
7397
|
+
}
|
|
7398
|
+
return digests.sort();
|
|
7399
|
+
}
|
|
7400
|
+
async function verifyShardSignature(input) {
|
|
7401
|
+
const { files, trustStore } = input;
|
|
7402
|
+
const sigBytes = files.get(SIGNATURE_ENTRY);
|
|
7403
|
+
if (!sigBytes) return { ok: false, reason: "unsigned" };
|
|
7404
|
+
if (sigBytes.byteLength > MAX_ENVELOPE_BYTES) {
|
|
7405
|
+
return { ok: false, reason: "malformed", detail: "signature envelope exceeds size cap" };
|
|
7406
|
+
}
|
|
7407
|
+
let envelope;
|
|
7408
|
+
try {
|
|
7409
|
+
const parsed = JSON.parse(new TextDecoder().decode(sigBytes));
|
|
7410
|
+
if (!isEnvelopeShape(parsed)) throw new Error("unexpected envelope shape");
|
|
7411
|
+
envelope = parsed;
|
|
7412
|
+
} catch (err) {
|
|
7413
|
+
return { ok: false, reason: "malformed", detail: err instanceof Error ? err.message : String(err) };
|
|
7414
|
+
}
|
|
7415
|
+
if (envelope.format_version !== SIGNING_ENVELOPE_VERSION) {
|
|
7416
|
+
return { ok: false, reason: "malformed", detail: `unsupported envelope version ${envelope.format_version}` };
|
|
7417
|
+
}
|
|
7418
|
+
if (envelope.signer.algorithm !== SIGNING_ALGORITHM) {
|
|
7419
|
+
return { ok: false, reason: "malformed", detail: `unsupported algorithm ${envelope.signer.algorithm}` };
|
|
7420
|
+
}
|
|
7421
|
+
const trusted = await trustStore.resolve(envelope.signer.key_id);
|
|
7422
|
+
if (!trusted) return { ok: false, reason: "unknown-signer", keyId: envelope.signer.key_id };
|
|
7423
|
+
if (trusted.revoked) return { ok: false, reason: "revoked", keyId: envelope.signer.key_id };
|
|
7424
|
+
if (trusted.public_key !== envelope.signer.public_key) {
|
|
7425
|
+
return { ok: false, reason: "bad-signature", keyId: envelope.signer.key_id };
|
|
7426
|
+
}
|
|
7427
|
+
if (!await isShardSigningSupported()) return { ok: false, reason: "unsupported" };
|
|
7428
|
+
const payload = {
|
|
7429
|
+
format_version: envelope.format_version,
|
|
7430
|
+
signer: envelope.signer,
|
|
7431
|
+
manifest_digest: envelope.manifest_digest,
|
|
7432
|
+
blob_digests: envelope.blob_digests
|
|
7433
|
+
};
|
|
7434
|
+
let signatureValid = false;
|
|
7435
|
+
try {
|
|
7436
|
+
const key = await globalThis.crypto.subtle.importKey(
|
|
7437
|
+
"raw",
|
|
7438
|
+
toBufferSource(base64urlToBytes(trusted.public_key)),
|
|
7439
|
+
{ name: "Ed25519" },
|
|
7440
|
+
false,
|
|
7441
|
+
["verify"]
|
|
7442
|
+
);
|
|
7443
|
+
const digest = await sha256Hex(canonicalPayloadBytes(payload));
|
|
7444
|
+
signatureValid = await globalThis.crypto.subtle.verify(
|
|
7445
|
+
"Ed25519",
|
|
7446
|
+
key,
|
|
7447
|
+
toBufferSource(base64urlToBytes(envelope.signature)),
|
|
7448
|
+
toBufferSource(new TextEncoder().encode(digest))
|
|
7449
|
+
);
|
|
7450
|
+
} catch (err) {
|
|
7451
|
+
return { ok: false, reason: "malformed", detail: err instanceof Error ? err.message : String(err) };
|
|
7452
|
+
}
|
|
7453
|
+
if (!signatureValid) return { ok: false, reason: "bad-signature", keyId: envelope.signer.key_id };
|
|
7454
|
+
const manifest = files.get("manifest.json");
|
|
7455
|
+
if (!manifest) {
|
|
7456
|
+
return { ok: false, reason: "content-mismatch", detail: "manifest.json missing" };
|
|
7457
|
+
}
|
|
7458
|
+
const manifestDigest = await sha256Hex(manifest);
|
|
7459
|
+
if (manifestDigest !== envelope.manifest_digest) {
|
|
7460
|
+
return { ok: false, reason: "content-mismatch", detail: "manifest digest does not match signature" };
|
|
7461
|
+
}
|
|
7462
|
+
const archiveBlobDigests = sidecarBlobDigests(files);
|
|
7463
|
+
const signedBlobDigests = [...envelope.blob_digests].sort();
|
|
7464
|
+
if (archiveBlobDigests.join(",") !== signedBlobDigests.join(",")) {
|
|
7465
|
+
return {
|
|
7466
|
+
ok: false,
|
|
7467
|
+
reason: "content-mismatch",
|
|
7468
|
+
detail: "sidecar blob digest set does not match signature"
|
|
7469
|
+
};
|
|
7470
|
+
}
|
|
7471
|
+
return { ok: true, keyId: envelope.signer.key_id };
|
|
7472
|
+
}
|
|
7473
|
+
function toBufferSource(bytes) {
|
|
7474
|
+
const copy = new Uint8Array(new ArrayBuffer(bytes.byteLength));
|
|
7475
|
+
copy.set(bytes);
|
|
7476
|
+
return copy;
|
|
7477
|
+
}
|
|
7478
|
+
function isEnvelopeShape(value) {
|
|
7479
|
+
if (typeof value !== "object" || value === null) return false;
|
|
7480
|
+
const v = value;
|
|
7481
|
+
const signer = v.signer;
|
|
7482
|
+
return typeof v.format_version === "string" && typeof v.manifest_digest === "string" && typeof v.signature === "string" && Array.isArray(v.blob_digests) && v.blob_digests.every((d) => typeof d === "string") && typeof signer === "object" && signer !== null && typeof signer.key_id === "string" && typeof signer.algorithm === "string" && typeof signer.public_key === "string";
|
|
7483
|
+
}
|
|
7484
|
+
async function signShard(input) {
|
|
7485
|
+
const manifest = input.files.get("manifest.json");
|
|
7486
|
+
if (!manifest) throw new Error("signShard: manifest.json missing from archive");
|
|
7487
|
+
const payload = {
|
|
7488
|
+
format_version: SIGNING_ENVELOPE_VERSION,
|
|
7489
|
+
signer: { key_id: input.keyId, algorithm: SIGNING_ALGORITHM, public_key: input.publicKey },
|
|
7490
|
+
manifest_digest: await sha256Hex(manifest),
|
|
7491
|
+
blob_digests: sidecarBlobDigests(input.files)
|
|
7492
|
+
};
|
|
7493
|
+
const digest = await sha256Hex(canonicalPayloadBytes(payload));
|
|
7494
|
+
const signature = await globalThis.crypto.subtle.sign(
|
|
7495
|
+
"Ed25519",
|
|
7496
|
+
input.privateKey,
|
|
7497
|
+
toBufferSource(new TextEncoder().encode(digest))
|
|
7498
|
+
);
|
|
7499
|
+
const envelope = {
|
|
7500
|
+
...payload,
|
|
7501
|
+
signature: bytesToBase64url(new Uint8Array(signature))
|
|
7502
|
+
};
|
|
7503
|
+
return new TextEncoder().encode(JSON.stringify(envelope, null, 2));
|
|
7504
|
+
}
|
|
7505
|
+
|
|
7014
7506
|
// src/shard/field-mapper.ts
|
|
7015
7507
|
function noteToShard(note) {
|
|
7016
7508
|
return {
|
|
@@ -7018,6 +7510,7 @@ function noteToShard(note) {
|
|
|
7018
7510
|
title: note.title,
|
|
7019
7511
|
original_content: note.original_content,
|
|
7020
7512
|
revised_content: note.revised_content,
|
|
7513
|
+
...note.ai_metadata != null ? { metadata: note.ai_metadata } : {},
|
|
7021
7514
|
collection_id: note.collection_id ?? null,
|
|
7022
7515
|
...note.attachments?.length ? { attachments: note.attachments } : {},
|
|
7023
7516
|
format: note.format,
|
|
@@ -7041,6 +7534,7 @@ function noteFromShard(shard) {
|
|
|
7041
7534
|
is_archived: shard.archived,
|
|
7042
7535
|
original_content: shard.original_content,
|
|
7043
7536
|
revised_content: shard.revised_content,
|
|
7537
|
+
ai_metadata: shard.metadata ?? null,
|
|
7044
7538
|
collection_id: shard.collection_id ?? null,
|
|
7045
7539
|
attachments,
|
|
7046
7540
|
tags: shard.tags,
|
|
@@ -7079,11 +7573,11 @@ function linkFromShard(shard) {
|
|
|
7079
7573
|
id: shard.id,
|
|
7080
7574
|
source_note_id: shard.from_note_id,
|
|
7081
7575
|
target_note_id: shard.to_note_id,
|
|
7082
|
-
to_url: shard.to_url,
|
|
7576
|
+
to_url: shard.to_url ?? null,
|
|
7083
7577
|
link_type: shard.kind,
|
|
7084
7578
|
confidence: shard.score,
|
|
7085
7579
|
created_at: shard.created_at,
|
|
7086
|
-
metadata: shard.metadata
|
|
7580
|
+
metadata: shard.metadata ?? null
|
|
7087
7581
|
};
|
|
7088
7582
|
}
|
|
7089
7583
|
function collectionToShard(collection, noteCount) {
|
|
@@ -7213,10 +7707,10 @@ function embeddingFromShard(shard) {
|
|
|
7213
7707
|
id: shard.id,
|
|
7214
7708
|
note_id: shard.note_id,
|
|
7215
7709
|
embedding_set_id: shard.embedding_set_id ?? null,
|
|
7216
|
-
chunk_index: shard.chunk_index,
|
|
7217
|
-
text: shard.text,
|
|
7710
|
+
chunk_index: shard.chunk_index ?? 0,
|
|
7711
|
+
text: shard.text ?? "",
|
|
7218
7712
|
vector: `[${shard.vector.join(",")}]`,
|
|
7219
|
-
model: shard.model,
|
|
7713
|
+
model: shard.model ?? null,
|
|
7220
7714
|
created_at: shard.created_at ?? null
|
|
7221
7715
|
};
|
|
7222
7716
|
}
|
|
@@ -7311,30 +7805,6 @@ function slugifyEmbeddingSet(value) {
|
|
|
7311
7805
|
return slug || "embedding-set";
|
|
7312
7806
|
}
|
|
7313
7807
|
|
|
7314
|
-
// src/shard/blob-sidecar.ts
|
|
7315
|
-
var SIDECAR_PREFIX = "blobs/";
|
|
7316
|
-
function blobChecksumToHex(checksum) {
|
|
7317
|
-
const sep = checksum.indexOf(":");
|
|
7318
|
-
return sep >= 0 ? checksum.slice(sep + 1) : checksum;
|
|
7319
|
-
}
|
|
7320
|
-
function sidecarEntryName(checksum) {
|
|
7321
|
-
return SIDECAR_PREFIX + blobChecksumToHex(checksum);
|
|
7322
|
-
}
|
|
7323
|
-
function isSidecarEntry(name) {
|
|
7324
|
-
if (!name.startsWith(SIDECAR_PREFIX)) return false;
|
|
7325
|
-
const rest = name.slice(SIDECAR_PREFIX.length);
|
|
7326
|
-
return rest.length > 0 && !rest.includes("/");
|
|
7327
|
-
}
|
|
7328
|
-
function collectSidecarBlobs(files) {
|
|
7329
|
-
const blobs = /* @__PURE__ */ new Map();
|
|
7330
|
-
for (const [name, bytes] of files) {
|
|
7331
|
-
if (isSidecarEntry(name)) {
|
|
7332
|
-
blobs.set(name.slice(SIDECAR_PREFIX.length), bytes);
|
|
7333
|
-
}
|
|
7334
|
-
}
|
|
7335
|
-
return blobs;
|
|
7336
|
-
}
|
|
7337
|
-
|
|
7338
7808
|
// src/shard/shard-export.ts
|
|
7339
7809
|
var encoder = new TextEncoder();
|
|
7340
7810
|
function jsonObject3(value) {
|
|
@@ -7356,6 +7826,7 @@ async function exportShard(db, options) {
|
|
|
7356
7826
|
n.created_at, n.updated_at, n.deleted_at,
|
|
7357
7827
|
o.content as original_content,
|
|
7358
7828
|
c.content as revised_content,
|
|
7829
|
+
c.ai_metadata,
|
|
7359
7830
|
$1::text as collection_id
|
|
7360
7831
|
FROM note n
|
|
7361
7832
|
LEFT JOIN note_original o ON o.note_id = n.id
|
|
@@ -7369,6 +7840,7 @@ async function exportShard(db, options) {
|
|
|
7369
7840
|
n.created_at, n.updated_at, n.deleted_at,
|
|
7370
7841
|
o.content as original_content,
|
|
7371
7842
|
c.content as revised_content,
|
|
7843
|
+
c.ai_metadata,
|
|
7372
7844
|
(
|
|
7373
7845
|
SELECT cn.collection_id
|
|
7374
7846
|
FROM collection_note cn
|
|
@@ -7388,6 +7860,7 @@ async function exportShard(db, options) {
|
|
|
7388
7860
|
n.created_at, n.updated_at, n.deleted_at,
|
|
7389
7861
|
o.content as original_content,
|
|
7390
7862
|
c.content as revised_content,
|
|
7863
|
+
c.ai_metadata,
|
|
7391
7864
|
(
|
|
7392
7865
|
SELECT cn.collection_id
|
|
7393
7866
|
FROM collection_note cn
|
|
@@ -7784,6 +8257,32 @@ function parseJsonArrayBytes(data) {
|
|
|
7784
8257
|
// src/shard/shard-import.ts
|
|
7785
8258
|
var decoder2 = new TextDecoder();
|
|
7786
8259
|
var DEFAULT_BATCH_SIZE = 250;
|
|
8260
|
+
async function enforceSignaturePolicy(files, options, warnings) {
|
|
8261
|
+
const policy = options?.verifySignature ?? (options?.trustStore ? "require" : void 0);
|
|
8262
|
+
if (!policy) return null;
|
|
8263
|
+
if (policy === "trusted-local-only") {
|
|
8264
|
+
warnings.push(
|
|
8265
|
+
"Signature verification skipped (trusted-local-only): the shard was imported without authenticating its publisher."
|
|
8266
|
+
);
|
|
8267
|
+
return null;
|
|
8268
|
+
}
|
|
8269
|
+
if (!options?.trustStore) {
|
|
8270
|
+
return `verifySignature: '${policy}' requires a trustStore to resolve signer keys.`;
|
|
8271
|
+
}
|
|
8272
|
+
const verdict = await verifyShardSignature({ files, trustStore: options.trustStore });
|
|
8273
|
+
if (verdict.ok) return null;
|
|
8274
|
+
if (verdict.reason === "unsigned") {
|
|
8275
|
+
if (policy === "prefer") {
|
|
8276
|
+
warnings.push(
|
|
8277
|
+
"Shard is unsigned; imported under verifySignature: prefer. Publisher provenance was NOT authenticated."
|
|
8278
|
+
);
|
|
8279
|
+
return null;
|
|
8280
|
+
}
|
|
8281
|
+
return "Shard is unsigned and verifySignature is required.";
|
|
8282
|
+
}
|
|
8283
|
+
const detail = "detail" in verdict ? `: ${verdict.detail}` : "keyId" in verdict ? ` (key ${verdict.keyId})` : "";
|
|
8284
|
+
return `Shard signature verification failed [${verdict.reason}]${detail}. No records or bytes were written.`;
|
|
8285
|
+
}
|
|
7787
8286
|
async function yieldToEventLoop2() {
|
|
7788
8287
|
const scheduler = globalThis.scheduler;
|
|
7789
8288
|
if (scheduler?.yield) {
|
|
@@ -7884,6 +8383,17 @@ async function importShard(db, data, options) {
|
|
|
7884
8383
|
duration_ms: performance.now() - start
|
|
7885
8384
|
};
|
|
7886
8385
|
}
|
|
8386
|
+
const sigError = await enforceSignaturePolicy(files, options, warnings);
|
|
8387
|
+
if (sigError) {
|
|
8388
|
+
return {
|
|
8389
|
+
success: false,
|
|
8390
|
+
counts,
|
|
8391
|
+
skipped,
|
|
8392
|
+
warnings,
|
|
8393
|
+
errors: [sigError],
|
|
8394
|
+
duration_ms: performance.now() - start
|
|
8395
|
+
};
|
|
8396
|
+
}
|
|
7887
8397
|
report?.({ phase: "validate", done: 0, total: 1 });
|
|
7888
8398
|
if (!manifest.checksums || typeof manifest.checksums !== "object") {
|
|
7889
8399
|
return {
|
|
@@ -8102,10 +8612,14 @@ async function importShard(db, data, options) {
|
|
|
8102
8612
|
);
|
|
8103
8613
|
}
|
|
8104
8614
|
await tx.query(
|
|
8105
|
-
`INSERT INTO note_revised_current (note_id, content)
|
|
8106
|
-
VALUES ($1, $2)
|
|
8107
|
-
ON CONFLICT (note_id) DO UPDATE SET content = $2`,
|
|
8108
|
-
[
|
|
8615
|
+
`INSERT INTO note_revised_current (note_id, content, ai_metadata)
|
|
8616
|
+
VALUES ($1, $2, $3::jsonb)
|
|
8617
|
+
ON CONFLICT (note_id) DO UPDATE SET content = $2, ai_metadata = $3::jsonb`,
|
|
8618
|
+
[
|
|
8619
|
+
note.id,
|
|
8620
|
+
note.revised_content ?? note.original_content,
|
|
8621
|
+
note.ai_metadata == null ? null : JSON.stringify(note.ai_metadata)
|
|
8622
|
+
]
|
|
8109
8623
|
);
|
|
8110
8624
|
} else {
|
|
8111
8625
|
await tx.query(
|
|
@@ -8129,9 +8643,13 @@ async function importShard(db, data, options) {
|
|
|
8129
8643
|
[generateId(), note.id, note.original_content, contentHash]
|
|
8130
8644
|
);
|
|
8131
8645
|
await tx.query(
|
|
8132
|
-
`INSERT INTO note_revised_current (note_id, content)
|
|
8133
|
-
VALUES ($1, $2) ${conflictClause}`,
|
|
8134
|
-
[
|
|
8646
|
+
`INSERT INTO note_revised_current (note_id, content, ai_metadata)
|
|
8647
|
+
VALUES ($1, $2, $3::jsonb) ${conflictClause}`,
|
|
8648
|
+
[
|
|
8649
|
+
note.id,
|
|
8650
|
+
note.revised_content ?? note.original_content,
|
|
8651
|
+
note.ai_metadata == null ? null : JSON.stringify(note.ai_metadata)
|
|
8652
|
+
]
|
|
8135
8653
|
);
|
|
8136
8654
|
}
|
|
8137
8655
|
for (const tag of note.tags) {
|
|
@@ -8636,8 +9154,8 @@ async function importShard(db, data, options) {
|
|
|
8636
9154
|
}
|
|
8637
9155
|
if (options?.blobStore && blobsToHydrate.size > 0) {
|
|
8638
9156
|
try {
|
|
8639
|
-
for (const [
|
|
8640
|
-
await options.blobStore.
|
|
9157
|
+
for (const [, bytes] of blobsToHydrate) {
|
|
9158
|
+
await options.blobStore.put(bytes);
|
|
8641
9159
|
}
|
|
8642
9160
|
} catch (err) {
|
|
8643
9161
|
warnings.push(
|
|
@@ -8655,13 +9173,14 @@ async function importShard(db, data, options) {
|
|
|
8655
9173
|
};
|
|
8656
9174
|
}
|
|
8657
9175
|
async function resolveEmbeddingSetIdForServerEmbedding(db, model, vector) {
|
|
9176
|
+
const modelName = model ?? "unknown";
|
|
8658
9177
|
const dimension = vectorDimension(vector);
|
|
8659
9178
|
const existing = await db.query(
|
|
8660
9179
|
`SELECT id FROM embedding_set
|
|
8661
9180
|
WHERE model_name = $1 AND dimensions = $2
|
|
8662
9181
|
ORDER BY created_at, id
|
|
8663
9182
|
LIMIT 1`,
|
|
8664
|
-
[
|
|
9183
|
+
[modelName, dimension]
|
|
8665
9184
|
);
|
|
8666
9185
|
if (existing.rows[0]?.id) return existing.rows[0].id;
|
|
8667
9186
|
const id = generateId();
|
|
@@ -8670,7 +9189,7 @@ async function resolveEmbeddingSetIdForServerEmbedding(db, model, vector) {
|
|
|
8670
9189
|
id, name, slug, description, purpose, document_count, embedding_count,
|
|
8671
9190
|
is_system, keywords_json, model_name, dimensions, kind, created_at, updated_at
|
|
8672
9191
|
) VALUES ($1, $2, $3, NULL, NULL, 0, 0, false, '[]'::jsonb, $2, $4, 'physical', now(), now())`,
|
|
8673
|
-
[id,
|
|
9192
|
+
[id, modelName, slugifyServerEmbeddingSet(modelName), dimension]
|
|
8674
9193
|
);
|
|
8675
9194
|
return id;
|
|
8676
9195
|
}
|
|
@@ -8889,7 +9408,7 @@ var knowledge_shard_schema_default = {
|
|
|
8889
9408
|
link: {
|
|
8890
9409
|
type: "object",
|
|
8891
9410
|
additionalProperties: false,
|
|
8892
|
-
required: ["id", "from_note_id", "to_note_id", "
|
|
9411
|
+
required: ["id", "from_note_id", "to_note_id", "kind", "score", "created_at"],
|
|
8893
9412
|
properties: {
|
|
8894
9413
|
id: { type: "string", minLength: 1 },
|
|
8895
9414
|
from_note_id: { type: "string", minLength: 1 },
|
|
@@ -8904,7 +9423,7 @@ var knowledge_shard_schema_default = {
|
|
|
8904
9423
|
embeddingSet: {
|
|
8905
9424
|
type: "object",
|
|
8906
9425
|
additionalProperties: false,
|
|
8907
|
-
required: ["id", "
|
|
9426
|
+
required: ["id", "model", "dimension"],
|
|
8908
9427
|
properties: {
|
|
8909
9428
|
id: { type: "string", minLength: 1 },
|
|
8910
9429
|
name: { type: "string" },
|
|
@@ -8932,10 +9451,11 @@ var knowledge_shard_schema_default = {
|
|
|
8932
9451
|
embeddingSetMember: {
|
|
8933
9452
|
type: "object",
|
|
8934
9453
|
additionalProperties: false,
|
|
8935
|
-
required: ["embedding_set_id", "note_id"
|
|
9454
|
+
required: ["embedding_set_id", "note_id"],
|
|
8936
9455
|
properties: {
|
|
8937
9456
|
embedding_set_id: { type: "string", minLength: 1 },
|
|
8938
9457
|
note_id: { type: "string", minLength: 1 },
|
|
9458
|
+
embedding_id: { type: "string", minLength: 1 },
|
|
8939
9459
|
membership_type: { type: "string" },
|
|
8940
9460
|
added_at: { $ref: "#/$defs/isoDateTime" },
|
|
8941
9461
|
added_by: { type: ["string", "null"] }
|
|
@@ -8959,7 +9479,7 @@ var knowledge_shard_schema_default = {
|
|
|
8959
9479
|
embedding: {
|
|
8960
9480
|
type: "object",
|
|
8961
9481
|
additionalProperties: false,
|
|
8962
|
-
required: ["id", "note_id", "
|
|
9482
|
+
required: ["id", "note_id", "vector"],
|
|
8963
9483
|
properties: {
|
|
8964
9484
|
id: { type: "string", minLength: 1 },
|
|
8965
9485
|
note_id: { type: "string", minLength: 1 },
|
|
@@ -9843,974 +10363,263 @@ function clearPrefetchedShard(url) {
|
|
|
9843
10363
|
}
|
|
9844
10364
|
warmStore.delete(url);
|
|
9845
10365
|
}
|
|
9846
|
-
|
|
9847
|
-
|
|
9848
|
-
|
|
9849
|
-
|
|
9850
|
-
|
|
9851
|
-
|
|
9852
|
-
|
|
9853
|
-
|
|
9854
|
-
|
|
9855
|
-
|
|
9856
|
-
|
|
9857
|
-
|
|
9858
|
-
|
|
9859
|
-
|
|
9860
|
-
|
|
9861
|
-
|
|
9862
|
-
|
|
9863
|
-
|
|
9864
|
-
|
|
9865
|
-
|
|
9866
|
-
|
|
9867
|
-
|
|
9868
|
-
|
|
9869
|
-
|
|
9870
|
-
|
|
9871
|
-
|
|
9872
|
-
|
|
9873
|
-
|
|
9874
|
-
|
|
9875
|
-
|
|
9876
|
-
|
|
9877
|
-
|
|
10366
|
+
var AIWG_SCAN_REQUIRED_FIELDS = [
|
|
10367
|
+
"schema_version",
|
|
10368
|
+
"id",
|
|
10369
|
+
"type",
|
|
10370
|
+
"title",
|
|
10371
|
+
"text",
|
|
10372
|
+
"facets",
|
|
10373
|
+
"tags",
|
|
10374
|
+
"concepts",
|
|
10375
|
+
"privacy"
|
|
10376
|
+
];
|
|
10377
|
+
function isPrivacyExcluded(record, options) {
|
|
10378
|
+
const privacy = record.privacy;
|
|
10379
|
+
if (!privacy || !isPrivacyClassification(privacy.classification) || typeof privacy.pii !== "boolean") return true;
|
|
10380
|
+
if (privacy.classification === "private" && !options?.includePrivate) return true;
|
|
10381
|
+
if (privacy.pii && !options?.includePii) return true;
|
|
10382
|
+
return false;
|
|
10383
|
+
}
|
|
10384
|
+
function filterAiwgRecordsByPrivacy(records, options) {
|
|
10385
|
+
return records.filter((record) => !isPrivacyExcluded(record, options));
|
|
10386
|
+
}
|
|
10387
|
+
var REQUIRED_RECORD_FIELDS = [
|
|
10388
|
+
"schema_version",
|
|
10389
|
+
"id",
|
|
10390
|
+
"type",
|
|
10391
|
+
"source",
|
|
10392
|
+
"facets",
|
|
10393
|
+
"tags",
|
|
10394
|
+
"concepts",
|
|
10395
|
+
"relationships",
|
|
10396
|
+
"provenance",
|
|
10397
|
+
"privacy",
|
|
10398
|
+
"updated_at"
|
|
10399
|
+
];
|
|
10400
|
+
var DEFAULT_QUERY_WEIGHTS = {
|
|
10401
|
+
title: 4,
|
|
10402
|
+
tag: 3,
|
|
10403
|
+
concept: 2,
|
|
10404
|
+
text: 1,
|
|
10405
|
+
facet: 2,
|
|
10406
|
+
id: 1,
|
|
10407
|
+
source: 0.25
|
|
10408
|
+
};
|
|
10409
|
+
function hasString(value) {
|
|
10410
|
+
return typeof value === "string" && value.length > 0;
|
|
10411
|
+
}
|
|
10412
|
+
function pushFacet(counts, name, value) {
|
|
10413
|
+
let bucket = counts[name];
|
|
10414
|
+
if (bucket === void 0) {
|
|
10415
|
+
bucket = /* @__PURE__ */ Object.create(null);
|
|
10416
|
+
counts[name] = bucket;
|
|
10417
|
+
}
|
|
10418
|
+
bucket[value] = (bucket[value] ?? 0) + 1;
|
|
10419
|
+
}
|
|
10420
|
+
function hasNonNegativeInteger(value) {
|
|
10421
|
+
return Number.isInteger(value) && typeof value === "number" && value >= 0;
|
|
10422
|
+
}
|
|
10423
|
+
function hasPositiveInteger(value) {
|
|
10424
|
+
return Number.isInteger(value) && typeof value === "number" && value > 0;
|
|
10425
|
+
}
|
|
10426
|
+
function isFacetCounts(value) {
|
|
10427
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
10428
|
+
return Object.values(value).every((counts) => !!counts && typeof counts === "object" && !Array.isArray(counts) && Object.values(counts).every((count) => hasNonNegativeInteger(count)));
|
|
10429
|
+
}
|
|
10430
|
+
function isPlainRecord(value) {
|
|
10431
|
+
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
10432
|
+
}
|
|
10433
|
+
function isOptionalStringArray(value) {
|
|
10434
|
+
return value === void 0 || Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
10435
|
+
}
|
|
10436
|
+
function isSupportedIndexSchemaVersion(value) {
|
|
10437
|
+
return value === "aiwg.fortemi.index.export.v1" || value === "aiwg.fortemi.index.export.v2";
|
|
10438
|
+
}
|
|
10439
|
+
function isSupportedRecordSchemaVersion(value) {
|
|
10440
|
+
return value === "aiwg.fortemi.index.record.v1" || value === "aiwg.fortemi.index.record.v2";
|
|
10441
|
+
}
|
|
10442
|
+
function validateOptionalRichMetadata(item, index, errors) {
|
|
10443
|
+
if (item.skos_concepts !== void 0) {
|
|
10444
|
+
if (!Array.isArray(item.skos_concepts)) {
|
|
10445
|
+
errors.push("items[" + index + "].skos_concepts must be an array when present");
|
|
10446
|
+
} else {
|
|
10447
|
+
for (const [conceptIndex, concept] of item.skos_concepts.entries()) {
|
|
10448
|
+
if (!isPlainRecord(concept)) {
|
|
10449
|
+
errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "] must be an object");
|
|
10450
|
+
continue;
|
|
9878
10451
|
}
|
|
9879
|
-
|
|
9880
|
-
|
|
9881
|
-
|
|
9882
|
-
|
|
9883
|
-
|
|
9884
|
-
required: ["previous_schema_version", "strategy"],
|
|
9885
|
-
properties: {
|
|
9886
|
-
previous_schema_version: {
|
|
9887
|
-
const: "aiwg.fortemi.index.export.v1"
|
|
9888
|
-
},
|
|
9889
|
-
strategy: {
|
|
9890
|
-
const: "supported"
|
|
10452
|
+
if (!hasString(concept.id)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].id is required");
|
|
10453
|
+
if (!hasString(concept.prefLabel)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].prefLabel is required");
|
|
10454
|
+
if (!isOptionalStringArray(concept.altLabels)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].altLabels must be a string array");
|
|
10455
|
+
if (concept.metadata !== void 0 && !isPlainRecord(concept.metadata)) {
|
|
10456
|
+
errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].metadata must be an object");
|
|
9891
10457
|
}
|
|
9892
10458
|
}
|
|
9893
|
-
},
|
|
9894
|
-
items: {
|
|
9895
|
-
type: "array",
|
|
9896
|
-
items: {
|
|
9897
|
-
$ref: "#/$defs/record"
|
|
9898
|
-
}
|
|
9899
10459
|
}
|
|
9900
|
-
}
|
|
9901
|
-
|
|
9902
|
-
{
|
|
9903
|
-
|
|
9904
|
-
|
|
9905
|
-
|
|
9906
|
-
|
|
9907
|
-
|
|
10460
|
+
}
|
|
10461
|
+
if (item.skos_relations !== void 0) {
|
|
10462
|
+
if (!Array.isArray(item.skos_relations)) {
|
|
10463
|
+
errors.push("items[" + index + "].skos_relations must be an array when present");
|
|
10464
|
+
} else {
|
|
10465
|
+
for (const [relationIndex, relation] of item.skos_relations.entries()) {
|
|
10466
|
+
if (!isPlainRecord(relation)) {
|
|
10467
|
+
errors.push("items[" + index + "].skos_relations[" + relationIndex + "] must be an object");
|
|
10468
|
+
continue;
|
|
9908
10469
|
}
|
|
9909
|
-
|
|
9910
|
-
|
|
9911
|
-
|
|
9912
|
-
|
|
9913
|
-
|
|
9914
|
-
properties: {
|
|
9915
|
-
source: {
|
|
9916
|
-
not: {
|
|
9917
|
-
required: ["graph"]
|
|
9918
|
-
}
|
|
9919
|
-
},
|
|
9920
|
-
items: {
|
|
9921
|
-
items: {
|
|
9922
|
-
allOf: [
|
|
9923
|
-
{
|
|
9924
|
-
properties: {
|
|
9925
|
-
schema_version: {
|
|
9926
|
-
const: "aiwg.fortemi.index.record.v1"
|
|
9927
|
-
}
|
|
9928
|
-
}
|
|
9929
|
-
},
|
|
9930
|
-
{
|
|
9931
|
-
$ref: "#/$defs/v1RecordCompatibility"
|
|
9932
|
-
}
|
|
9933
|
-
]
|
|
9934
|
-
}
|
|
9935
|
-
}
|
|
10470
|
+
if (!hasString(relation.type)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].type is required");
|
|
10471
|
+
if (!hasString(relation.source_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].source_id is required");
|
|
10472
|
+
if (!hasString(relation.target_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].target_id is required");
|
|
10473
|
+
if (relation.metadata !== void 0 && !isPlainRecord(relation.metadata)) {
|
|
10474
|
+
errors.push("items[" + index + "].skos_relations[" + relationIndex + "].metadata must be an object");
|
|
9936
10475
|
}
|
|
9937
10476
|
}
|
|
9938
|
-
}
|
|
9939
|
-
|
|
9940
|
-
|
|
9941
|
-
|
|
9942
|
-
|
|
9943
|
-
|
|
9944
|
-
|
|
10477
|
+
}
|
|
10478
|
+
}
|
|
10479
|
+
if (item.provenance_events !== void 0) {
|
|
10480
|
+
if (!Array.isArray(item.provenance_events)) {
|
|
10481
|
+
errors.push("items[" + index + "].provenance_events must be an array when present");
|
|
10482
|
+
} else {
|
|
10483
|
+
for (const [eventIndex, event] of item.provenance_events.entries()) {
|
|
10484
|
+
if (!isPlainRecord(event)) {
|
|
10485
|
+
errors.push("items[" + index + "].provenance_events[" + eventIndex + "] must be an object");
|
|
10486
|
+
continue;
|
|
9945
10487
|
}
|
|
9946
|
-
|
|
9947
|
-
|
|
9948
|
-
|
|
9949
|
-
properties: {
|
|
9950
|
-
source: {
|
|
9951
|
-
required: ["graph"]
|
|
9952
|
-
},
|
|
9953
|
-
items: {
|
|
9954
|
-
items: {
|
|
9955
|
-
properties: {
|
|
9956
|
-
schema_version: {
|
|
9957
|
-
const: "aiwg.fortemi.index.record.v2"
|
|
9958
|
-
}
|
|
9959
|
-
}
|
|
9960
|
-
}
|
|
9961
|
-
}
|
|
10488
|
+
if (!hasString(event.activity)) errors.push("items[" + index + "].provenance_events[" + eventIndex + "].activity is required");
|
|
10489
|
+
if (event.attributes !== void 0 && !isPlainRecord(event.attributes)) {
|
|
10490
|
+
errors.push("items[" + index + "].provenance_events[" + eventIndex + "].attributes must be an object");
|
|
9962
10491
|
}
|
|
9963
10492
|
}
|
|
9964
10493
|
}
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
type: "string",
|
|
9972
|
-
minLength: 1
|
|
9973
|
-
},
|
|
9974
|
-
stringArray: {
|
|
9975
|
-
type: "array",
|
|
9976
|
-
items: {
|
|
9977
|
-
type: "string"
|
|
10494
|
+
}
|
|
10495
|
+
if (Array.isArray(item.relationships)) {
|
|
10496
|
+
for (const [relationshipIndex, relationship] of item.relationships.entries()) {
|
|
10497
|
+
if (!isPlainRecord(relationship)) {
|
|
10498
|
+
errors.push("items[" + index + "].relationships[" + relationshipIndex + "] must be an object");
|
|
10499
|
+
continue;
|
|
9978
10500
|
}
|
|
9979
|
-
|
|
9980
|
-
|
|
9981
|
-
type: "array",
|
|
9982
|
-
items: {
|
|
9983
|
-
type: "number"
|
|
10501
|
+
if (relationship.metadata !== void 0 && !isPlainRecord(relationship.metadata)) {
|
|
10502
|
+
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].metadata must be an object");
|
|
9984
10503
|
}
|
|
9985
|
-
|
|
9986
|
-
|
|
9987
|
-
|
|
9988
|
-
|
|
9989
|
-
|
|
9990
|
-
|
|
9991
|
-
|
|
9992
|
-
|
|
9993
|
-
|
|
9994
|
-
|
|
9995
|
-
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10002
|
-
|
|
10003
|
-
|
|
10004
|
-
|
|
10005
|
-
|
|
10006
|
-
|
|
10007
|
-
|
|
10008
|
-
|
|
10009
|
-
|
|
10010
|
-
|
|
10011
|
-
|
|
10012
|
-
|
|
10013
|
-
|
|
10014
|
-
|
|
10015
|
-
|
|
10016
|
-
}
|
|
10017
|
-
]
|
|
10018
|
-
},
|
|
10019
|
-
properties: {
|
|
10020
|
-
source: {
|
|
10021
|
-
not: {
|
|
10022
|
-
anyOf: [
|
|
10023
|
-
{
|
|
10024
|
-
required: ["origin"]
|
|
10025
|
-
},
|
|
10026
|
-
{
|
|
10027
|
-
required: ["generated"]
|
|
10028
|
-
},
|
|
10029
|
-
{
|
|
10030
|
-
required: ["checksum"]
|
|
10031
|
-
},
|
|
10032
|
-
{
|
|
10033
|
-
required: ["updated_at"]
|
|
10034
|
-
}
|
|
10035
|
-
]
|
|
10036
|
-
}
|
|
10037
|
-
},
|
|
10038
|
-
relationships: {
|
|
10039
|
-
items: {
|
|
10040
|
-
not: {
|
|
10041
|
-
anyOf: [
|
|
10042
|
-
{
|
|
10043
|
-
required: ["target_path"]
|
|
10044
|
-
},
|
|
10045
|
-
{
|
|
10046
|
-
required: ["direction"]
|
|
10047
|
-
},
|
|
10048
|
-
{
|
|
10049
|
-
required: ["label"]
|
|
10050
|
-
},
|
|
10051
|
-
{
|
|
10052
|
-
required: ["confidence"]
|
|
10053
|
-
},
|
|
10054
|
-
{
|
|
10055
|
-
required: ["privacy"]
|
|
10056
|
-
},
|
|
10057
|
-
{
|
|
10058
|
-
required: ["metadata"]
|
|
10059
|
-
}
|
|
10060
|
-
]
|
|
10061
|
-
}
|
|
10062
|
-
}
|
|
10063
|
-
},
|
|
10064
|
-
privacy: {
|
|
10065
|
-
not: {
|
|
10066
|
-
required: ["locality"]
|
|
10067
|
-
}
|
|
10504
|
+
if (relationship.direction !== void 0 && relationship.direction !== "upstream" && relationship.direction !== "downstream" && relationship.direction !== "related") {
|
|
10505
|
+
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].direction must be upstream, downstream, or related");
|
|
10506
|
+
}
|
|
10507
|
+
if (relationship.target_path !== void 0 && typeof relationship.target_path !== "string") {
|
|
10508
|
+
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].target_path must be a string");
|
|
10509
|
+
}
|
|
10510
|
+
}
|
|
10511
|
+
}
|
|
10512
|
+
if (item.search !== void 0) {
|
|
10513
|
+
if (!isPlainRecord(item.search)) {
|
|
10514
|
+
errors.push("items[" + index + "].search must be an object");
|
|
10515
|
+
} else {
|
|
10516
|
+
if (!isOptionalStringArray(item.search.triggers)) errors.push("items[" + index + "].search.triggers must be a string array");
|
|
10517
|
+
if (!isOptionalStringArray(item.search.aliases)) errors.push("items[" + index + "].search.aliases must be a string array");
|
|
10518
|
+
if (!isOptionalStringArray(item.search.tags)) errors.push("items[" + index + "].search.tags must be a string array");
|
|
10519
|
+
if (item.search.frontmatter !== void 0 && !isPlainRecord(item.search.frontmatter)) {
|
|
10520
|
+
errors.push("items[" + index + "].search.frontmatter must be an object");
|
|
10521
|
+
}
|
|
10522
|
+
}
|
|
10523
|
+
}
|
|
10524
|
+
if (item.chunks !== void 0) {
|
|
10525
|
+
if (!Array.isArray(item.chunks)) {
|
|
10526
|
+
errors.push("items[" + index + "].chunks must be an array when present");
|
|
10527
|
+
} else {
|
|
10528
|
+
for (const [chunkIndex, chunk] of item.chunks.entries()) {
|
|
10529
|
+
if (!isPlainRecord(chunk)) {
|
|
10530
|
+
errors.push("items[" + index + "].chunks[" + chunkIndex + "] must be an object");
|
|
10531
|
+
continue;
|
|
10532
|
+
}
|
|
10533
|
+
if (chunk.metadata !== void 0 && !isPlainRecord(chunk.metadata)) {
|
|
10534
|
+
errors.push("items[" + index + "].chunks[" + chunkIndex + "].metadata must be an object");
|
|
10068
10535
|
}
|
|
10069
10536
|
}
|
|
10070
|
-
}
|
|
10071
|
-
|
|
10072
|
-
|
|
10073
|
-
|
|
10074
|
-
|
|
10075
|
-
|
|
10076
|
-
|
|
10077
|
-
|
|
10078
|
-
|
|
10079
|
-
|
|
10080
|
-
|
|
10081
|
-
|
|
10082
|
-
"
|
|
10083
|
-
|
|
10084
|
-
|
|
10085
|
-
|
|
10086
|
-
|
|
10087
|
-
"updated_at"
|
|
10088
|
-
],
|
|
10089
|
-
properties: {
|
|
10090
|
-
schema_version: {
|
|
10091
|
-
enum: [
|
|
10092
|
-
"aiwg.fortemi.index.record.v1",
|
|
10093
|
-
"aiwg.fortemi.index.record.v2"
|
|
10094
|
-
]
|
|
10095
|
-
},
|
|
10096
|
-
id: {
|
|
10097
|
-
type: "string",
|
|
10098
|
-
minLength: 1
|
|
10099
|
-
},
|
|
10100
|
-
type: {
|
|
10101
|
-
$ref: "#/$defs/recordType"
|
|
10102
|
-
},
|
|
10103
|
-
source: {
|
|
10104
|
-
type: "object",
|
|
10105
|
-
additionalProperties: false,
|
|
10106
|
-
required: ["path", "repo_relative_path", "locator"],
|
|
10107
|
-
properties: {
|
|
10108
|
-
path: {
|
|
10109
|
-
type: "string",
|
|
10110
|
-
minLength: 1
|
|
10111
|
-
},
|
|
10112
|
-
repo_relative_path: {
|
|
10113
|
-
type: "string",
|
|
10114
|
-
minLength: 1
|
|
10115
|
-
},
|
|
10116
|
-
locator: {
|
|
10117
|
-
type: "string",
|
|
10118
|
-
minLength: 1
|
|
10119
|
-
},
|
|
10120
|
-
origin: {
|
|
10121
|
-
type: "string",
|
|
10122
|
-
minLength: 1
|
|
10123
|
-
},
|
|
10124
|
-
generated: {
|
|
10125
|
-
type: "boolean"
|
|
10126
|
-
},
|
|
10127
|
-
checksum: {
|
|
10128
|
-
type: "string"
|
|
10129
|
-
},
|
|
10130
|
-
updated_at: {
|
|
10131
|
-
type: "string",
|
|
10132
|
-
format: "date-time"
|
|
10133
|
-
}
|
|
10134
|
-
}
|
|
10135
|
-
},
|
|
10136
|
-
title: {
|
|
10137
|
-
type: "string"
|
|
10138
|
-
},
|
|
10139
|
-
name: {
|
|
10140
|
-
type: "string"
|
|
10141
|
-
},
|
|
10142
|
-
summary: {
|
|
10143
|
-
type: "string"
|
|
10144
|
-
},
|
|
10145
|
-
text: {
|
|
10146
|
-
type: "string"
|
|
10147
|
-
},
|
|
10148
|
-
search: {
|
|
10149
|
-
type: "object",
|
|
10150
|
-
additionalProperties: false,
|
|
10151
|
-
required: [
|
|
10152
|
-
"title",
|
|
10153
|
-
"body",
|
|
10154
|
-
"triggers",
|
|
10155
|
-
"aliases",
|
|
10156
|
-
"tags",
|
|
10157
|
-
"frontmatter"
|
|
10158
|
-
],
|
|
10159
|
-
properties: {
|
|
10160
|
-
title: {
|
|
10161
|
-
type: "string"
|
|
10162
|
-
},
|
|
10163
|
-
name: {
|
|
10164
|
-
type: "string"
|
|
10165
|
-
},
|
|
10166
|
-
summary: {
|
|
10167
|
-
type: "string"
|
|
10168
|
-
},
|
|
10169
|
-
body: {
|
|
10170
|
-
type: "string"
|
|
10171
|
-
},
|
|
10172
|
-
triggers: {
|
|
10173
|
-
$ref: "#/$defs/stringArray"
|
|
10174
|
-
},
|
|
10175
|
-
aliases: {
|
|
10176
|
-
$ref: "#/$defs/stringArray"
|
|
10177
|
-
},
|
|
10178
|
-
capability: {
|
|
10179
|
-
type: "string"
|
|
10180
|
-
},
|
|
10181
|
-
tags: {
|
|
10182
|
-
$ref: "#/$defs/stringArray"
|
|
10183
|
-
},
|
|
10184
|
-
phase: {
|
|
10185
|
-
type: "string"
|
|
10186
|
-
},
|
|
10187
|
-
type: {
|
|
10188
|
-
type: "string"
|
|
10189
|
-
},
|
|
10190
|
-
frontmatter: {
|
|
10191
|
-
type: "object"
|
|
10192
|
-
}
|
|
10193
|
-
}
|
|
10194
|
-
},
|
|
10195
|
-
facets: {
|
|
10196
|
-
type: "object",
|
|
10197
|
-
additionalProperties: {
|
|
10198
|
-
$ref: "#/$defs/stringArray"
|
|
10199
|
-
}
|
|
10200
|
-
},
|
|
10201
|
-
tags: {
|
|
10202
|
-
$ref: "#/$defs/stringArray"
|
|
10203
|
-
},
|
|
10204
|
-
concepts: {
|
|
10205
|
-
$ref: "#/$defs/stringArray"
|
|
10206
|
-
},
|
|
10207
|
-
relationships: {
|
|
10208
|
-
type: "array",
|
|
10209
|
-
items: {
|
|
10210
|
-
type: "object",
|
|
10211
|
-
additionalProperties: false,
|
|
10212
|
-
required: ["type", "target_id"],
|
|
10213
|
-
properties: {
|
|
10214
|
-
type: {
|
|
10215
|
-
type: "string",
|
|
10216
|
-
minLength: 1
|
|
10217
|
-
},
|
|
10218
|
-
target_id: {
|
|
10219
|
-
type: "string",
|
|
10220
|
-
minLength: 1
|
|
10221
|
-
},
|
|
10222
|
-
source_path: {
|
|
10223
|
-
type: "string"
|
|
10224
|
-
},
|
|
10225
|
-
target_path: {
|
|
10226
|
-
type: "string"
|
|
10227
|
-
},
|
|
10228
|
-
direction: {
|
|
10229
|
-
enum: ["upstream", "downstream", "related"]
|
|
10230
|
-
},
|
|
10231
|
-
label: {
|
|
10232
|
-
type: "string"
|
|
10233
|
-
},
|
|
10234
|
-
confidence: {
|
|
10235
|
-
type: "number"
|
|
10236
|
-
},
|
|
10237
|
-
privacy: {
|
|
10238
|
-
$ref: "#/$defs/privacy"
|
|
10239
|
-
},
|
|
10240
|
-
metadata: {
|
|
10241
|
-
type: "object"
|
|
10242
|
-
}
|
|
10243
|
-
}
|
|
10244
|
-
}
|
|
10245
|
-
},
|
|
10246
|
-
provenance: {
|
|
10247
|
-
type: "array",
|
|
10248
|
-
minItems: 1,
|
|
10249
|
-
items: {
|
|
10250
|
-
type: "object",
|
|
10251
|
-
additionalProperties: false,
|
|
10252
|
-
required: ["field", "source", "path", "confidence", "privacy"],
|
|
10253
|
-
properties: {
|
|
10254
|
-
field: {
|
|
10255
|
-
type: "string",
|
|
10256
|
-
minLength: 1
|
|
10257
|
-
},
|
|
10258
|
-
source: {
|
|
10259
|
-
type: "string",
|
|
10260
|
-
minLength: 1
|
|
10261
|
-
},
|
|
10262
|
-
path: {
|
|
10263
|
-
type: "string",
|
|
10264
|
-
minLength: 1
|
|
10265
|
-
},
|
|
10266
|
-
confidence: {
|
|
10267
|
-
enum: ["source", "candidate", "reviewed", "rejected"]
|
|
10268
|
-
},
|
|
10269
|
-
privacy: {
|
|
10270
|
-
$ref: "#/$defs/privacy"
|
|
10271
|
-
}
|
|
10272
|
-
}
|
|
10273
|
-
}
|
|
10274
|
-
},
|
|
10275
|
-
privacy: {
|
|
10276
|
-
type: "object",
|
|
10277
|
-
additionalProperties: false,
|
|
10278
|
-
required: ["classification", "pii"],
|
|
10279
|
-
properties: {
|
|
10280
|
-
classification: {
|
|
10281
|
-
$ref: "#/$defs/privacy"
|
|
10282
|
-
},
|
|
10283
|
-
pii: {
|
|
10284
|
-
type: "boolean"
|
|
10285
|
-
},
|
|
10286
|
-
locality: {
|
|
10287
|
-
enum: ["project", "framework", "external"]
|
|
10288
|
-
}
|
|
10289
|
-
}
|
|
10290
|
-
},
|
|
10291
|
-
chunks: {
|
|
10292
|
-
type: "array",
|
|
10293
|
-
items: {
|
|
10294
|
-
type: "object",
|
|
10295
|
-
additionalProperties: false,
|
|
10296
|
-
properties: {
|
|
10297
|
-
id: {
|
|
10298
|
-
type: "string",
|
|
10299
|
-
minLength: 1
|
|
10300
|
-
},
|
|
10301
|
-
text: {
|
|
10302
|
-
type: "string"
|
|
10303
|
-
},
|
|
10304
|
-
body: {
|
|
10305
|
-
type: "string"
|
|
10306
|
-
},
|
|
10307
|
-
summary: {
|
|
10308
|
-
type: "string"
|
|
10309
|
-
},
|
|
10310
|
-
source_path: {
|
|
10311
|
-
type: "string"
|
|
10312
|
-
},
|
|
10313
|
-
metadata: {
|
|
10314
|
-
type: "object"
|
|
10315
|
-
},
|
|
10316
|
-
checksum: {
|
|
10317
|
-
type: "string"
|
|
10318
|
-
}
|
|
10319
|
-
}
|
|
10320
|
-
}
|
|
10321
|
-
},
|
|
10322
|
-
embeddings: {
|
|
10323
|
-
type: "array",
|
|
10324
|
-
items: {
|
|
10325
|
-
type: "object",
|
|
10326
|
-
additionalProperties: false,
|
|
10327
|
-
properties: {
|
|
10328
|
-
id: {
|
|
10329
|
-
type: "string"
|
|
10330
|
-
},
|
|
10331
|
-
model: {
|
|
10332
|
-
type: "string"
|
|
10333
|
-
},
|
|
10334
|
-
embedding: {
|
|
10335
|
-
$ref: "#/$defs/numberArray"
|
|
10336
|
-
},
|
|
10337
|
-
vector: {
|
|
10338
|
-
$ref: "#/$defs/numberArray"
|
|
10339
|
-
},
|
|
10340
|
-
granularity: {
|
|
10341
|
-
type: "string"
|
|
10342
|
-
},
|
|
10343
|
-
source_path: {
|
|
10344
|
-
type: "string"
|
|
10345
|
-
},
|
|
10346
|
-
metadata: {
|
|
10347
|
-
type: "object"
|
|
10348
|
-
},
|
|
10349
|
-
chunk_id: {
|
|
10350
|
-
type: "string"
|
|
10351
|
-
},
|
|
10352
|
-
vector_ref: {
|
|
10353
|
-
type: "string"
|
|
10354
|
-
},
|
|
10355
|
-
input_hash: {
|
|
10356
|
-
type: "string"
|
|
10357
|
-
}
|
|
10358
|
-
}
|
|
10359
|
-
}
|
|
10360
|
-
},
|
|
10361
|
-
compatibility: {
|
|
10362
|
-
type: "object"
|
|
10363
|
-
},
|
|
10364
|
-
skos_concepts: {
|
|
10365
|
-
type: "array",
|
|
10366
|
-
items: {
|
|
10367
|
-
type: "object",
|
|
10368
|
-
additionalProperties: false,
|
|
10369
|
-
required: ["id", "prefLabel"],
|
|
10370
|
-
properties: {
|
|
10371
|
-
id: {
|
|
10372
|
-
type: "string",
|
|
10373
|
-
minLength: 1
|
|
10374
|
-
},
|
|
10375
|
-
prefLabel: {
|
|
10376
|
-
type: "string",
|
|
10377
|
-
minLength: 1
|
|
10378
|
-
},
|
|
10379
|
-
definition: {
|
|
10380
|
-
type: "string"
|
|
10381
|
-
},
|
|
10382
|
-
scheme: {
|
|
10383
|
-
type: "string"
|
|
10384
|
-
},
|
|
10385
|
-
notation: {
|
|
10386
|
-
type: "string"
|
|
10387
|
-
},
|
|
10388
|
-
uri: {
|
|
10389
|
-
type: "string"
|
|
10390
|
-
},
|
|
10391
|
-
altLabels: {
|
|
10392
|
-
$ref: "#/$defs/stringArray"
|
|
10393
|
-
},
|
|
10394
|
-
metadata: {
|
|
10395
|
-
type: "object"
|
|
10396
|
-
}
|
|
10397
|
-
}
|
|
10398
|
-
}
|
|
10399
|
-
},
|
|
10400
|
-
skos_relations: {
|
|
10401
|
-
type: "array",
|
|
10402
|
-
items: {
|
|
10403
|
-
type: "object",
|
|
10404
|
-
additionalProperties: false,
|
|
10405
|
-
required: ["type", "source_id", "target_id"],
|
|
10406
|
-
properties: {
|
|
10407
|
-
type: {
|
|
10408
|
-
type: "string",
|
|
10409
|
-
minLength: 1
|
|
10410
|
-
},
|
|
10411
|
-
source_id: {
|
|
10412
|
-
type: "string",
|
|
10413
|
-
minLength: 1
|
|
10414
|
-
},
|
|
10415
|
-
target_id: {
|
|
10416
|
-
type: "string",
|
|
10417
|
-
minLength: 1
|
|
10418
|
-
},
|
|
10419
|
-
source_path: {
|
|
10420
|
-
type: "string"
|
|
10421
|
-
},
|
|
10422
|
-
metadata: {
|
|
10423
|
-
type: "object"
|
|
10424
|
-
}
|
|
10425
|
-
}
|
|
10426
|
-
}
|
|
10427
|
-
},
|
|
10428
|
-
provenance_events: {
|
|
10429
|
-
type: "array",
|
|
10430
|
-
items: {
|
|
10431
|
-
type: "object",
|
|
10432
|
-
additionalProperties: false,
|
|
10433
|
-
required: ["activity"],
|
|
10434
|
-
properties: {
|
|
10435
|
-
id: {
|
|
10436
|
-
type: "string"
|
|
10437
|
-
},
|
|
10438
|
-
activity: {
|
|
10439
|
-
type: "string",
|
|
10440
|
-
minLength: 1
|
|
10441
|
-
},
|
|
10442
|
-
agent: {
|
|
10443
|
-
type: "string"
|
|
10444
|
-
},
|
|
10445
|
-
started_at: {
|
|
10446
|
-
type: "string",
|
|
10447
|
-
format: "date-time"
|
|
10448
|
-
},
|
|
10449
|
-
ended_at: {
|
|
10450
|
-
type: "string",
|
|
10451
|
-
format: "date-time"
|
|
10452
|
-
},
|
|
10453
|
-
source: {
|
|
10454
|
-
type: "string"
|
|
10455
|
-
},
|
|
10456
|
-
path: {
|
|
10457
|
-
type: "string"
|
|
10458
|
-
},
|
|
10459
|
-
confidence: {
|
|
10460
|
-
enum: ["source", "candidate", "reviewed", "rejected"]
|
|
10461
|
-
},
|
|
10462
|
-
privacy: {
|
|
10463
|
-
$ref: "#/$defs/privacy"
|
|
10464
|
-
},
|
|
10465
|
-
attributes: {
|
|
10466
|
-
type: "object"
|
|
10467
|
-
}
|
|
10468
|
-
}
|
|
10469
|
-
}
|
|
10470
|
-
},
|
|
10471
|
-
updated_at: {
|
|
10472
|
-
type: "string",
|
|
10473
|
-
format: "date-time"
|
|
10537
|
+
}
|
|
10538
|
+
}
|
|
10539
|
+
if (item.embeddings !== void 0) {
|
|
10540
|
+
if (!Array.isArray(item.embeddings)) {
|
|
10541
|
+
errors.push("items[" + index + "].embeddings must be an array when present");
|
|
10542
|
+
} else {
|
|
10543
|
+
for (const [embeddingIndex, embedding] of item.embeddings.entries()) {
|
|
10544
|
+
if (!isPlainRecord(embedding)) {
|
|
10545
|
+
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "] must be an object");
|
|
10546
|
+
continue;
|
|
10547
|
+
}
|
|
10548
|
+
const vector = embedding.embedding ?? embedding.vector;
|
|
10549
|
+
if (vector !== void 0 && (!Array.isArray(vector) || !vector.every((entry) => typeof entry === "number"))) {
|
|
10550
|
+
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "].embedding/vector must be a number array");
|
|
10551
|
+
}
|
|
10552
|
+
if (embedding.metadata !== void 0 && !isPlainRecord(embedding.metadata)) {
|
|
10553
|
+
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "].metadata must be an object");
|
|
10474
10554
|
}
|
|
10475
10555
|
}
|
|
10476
10556
|
}
|
|
10477
10557
|
}
|
|
10478
|
-
|
|
10479
|
-
|
|
10480
|
-
// src/aiwg-index-schema.ts
|
|
10481
|
-
var PROJECTED_FIELDS = [
|
|
10482
|
-
"schema_version",
|
|
10483
|
-
"id",
|
|
10484
|
-
"type",
|
|
10485
|
-
"title",
|
|
10486
|
-
"text",
|
|
10487
|
-
"facets",
|
|
10488
|
-
"tags",
|
|
10489
|
-
"concepts",
|
|
10490
|
-
"privacy"
|
|
10491
|
-
];
|
|
10492
|
-
var ajvInstance2;
|
|
10493
|
-
var exportValidator;
|
|
10494
|
-
var projectedRecordValidator;
|
|
10495
|
-
function getAiwgFortemiIndexExportSchema() {
|
|
10496
|
-
return aiwg_fortemi_index_export_schema_default;
|
|
10497
|
-
}
|
|
10498
|
-
function getAjv2() {
|
|
10499
|
-
if (!ajvInstance2) {
|
|
10500
|
-
ajvInstance2 = new Ajv2020({
|
|
10501
|
-
allErrors: true,
|
|
10502
|
-
strict: false,
|
|
10503
|
-
validateFormats: false
|
|
10504
|
-
});
|
|
10505
|
-
ajvInstance2.addSchema(aiwg_fortemi_index_export_schema_default);
|
|
10558
|
+
if (item.compatibility !== void 0 && !isPlainRecord(item.compatibility)) {
|
|
10559
|
+
errors.push("items[" + index + "].compatibility must be an object");
|
|
10506
10560
|
}
|
|
10507
|
-
return ajvInstance2;
|
|
10508
10561
|
}
|
|
10509
|
-
function
|
|
10510
|
-
return
|
|
10511
|
-
const path = error.instancePath || "(root)";
|
|
10512
|
-
return `${path} ${error.message ?? "is invalid"}`;
|
|
10513
|
-
});
|
|
10562
|
+
function isPrivacyClassification(value) {
|
|
10563
|
+
return value === "private" || value === "sanitized" || value === "public";
|
|
10514
10564
|
}
|
|
10515
|
-
function
|
|
10516
|
-
|
|
10517
|
-
return exportValidator;
|
|
10565
|
+
function isProvenanceConfidence(value) {
|
|
10566
|
+
return value === "source" || value === "candidate" || value === "reviewed" || value === "rejected";
|
|
10518
10567
|
}
|
|
10519
|
-
function
|
|
10520
|
-
if (!
|
|
10521
|
-
|
|
10522
|
-
|
|
10523
|
-
|
|
10524
|
-
|
|
10525
|
-
|
|
10526
|
-
|
|
10527
|
-
|
|
10528
|
-
|
|
10529
|
-
|
|
10530
|
-
|
|
10531
|
-
|
|
10532
|
-
if: {
|
|
10533
|
-
properties: {
|
|
10534
|
-
schema_version: { const: "aiwg.fortemi.index.record.v1" }
|
|
10535
|
-
}
|
|
10536
|
-
},
|
|
10537
|
-
then: { $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/v1RecordCompatibility` }
|
|
10538
|
-
}
|
|
10539
|
-
]
|
|
10540
|
-
});
|
|
10568
|
+
function validateProvenanceItems(item, index, errors) {
|
|
10569
|
+
if (!Array.isArray(item.provenance)) return;
|
|
10570
|
+
for (const [provIndex, prov] of item.provenance.entries()) {
|
|
10571
|
+
const at = "items[" + index + "].provenance[" + provIndex + "]";
|
|
10572
|
+
if (!isPlainRecord(prov)) {
|
|
10573
|
+
errors.push(at + " must be an object");
|
|
10574
|
+
continue;
|
|
10575
|
+
}
|
|
10576
|
+
if (!hasString(prov.field)) errors.push(at + ".field is required");
|
|
10577
|
+
if (!hasString(prov.source)) errors.push(at + ".source is required");
|
|
10578
|
+
if (!hasString(prov.path)) errors.push(at + ".path is required");
|
|
10579
|
+
if (!isProvenanceConfidence(prov.confidence)) errors.push(at + ".confidence must be one of source, candidate, reviewed, rejected");
|
|
10580
|
+
if (!isPrivacyClassification(prov.privacy)) errors.push(at + ".privacy must be one of private, sanitized, public");
|
|
10541
10581
|
}
|
|
10542
|
-
return projectedRecordValidator;
|
|
10543
|
-
}
|
|
10544
|
-
function validateAiwgFortemiIndexExportSchema(value) {
|
|
10545
|
-
const validate = getExportValidator();
|
|
10546
|
-
const schemaValue = value && typeof value === "object" && Array.isArray(value.items) ? {
|
|
10547
|
-
...value,
|
|
10548
|
-
items: value.items.map((item) => {
|
|
10549
|
-
if (!item || typeof item !== "object" || Array.isArray(item)) return item;
|
|
10550
|
-
const schemaRecord = { ...item };
|
|
10551
|
-
Reflect.deleteProperty(schemaRecord, "binary_sources");
|
|
10552
|
-
return schemaRecord;
|
|
10553
|
-
})
|
|
10554
|
-
} : value;
|
|
10555
|
-
const valid = validate(schemaValue);
|
|
10556
|
-
return { valid, errors: formatErrors2(validate.errors) };
|
|
10557
|
-
}
|
|
10558
|
-
function validateAiwgFortemiProjectedRecordSchema(value) {
|
|
10559
|
-
const validate = getProjectedRecordValidator();
|
|
10560
|
-
const valid = validate(value);
|
|
10561
|
-
return { valid, errors: formatErrors2(validate.errors) };
|
|
10562
|
-
}
|
|
10563
|
-
|
|
10564
|
-
// src/aiwg-index.ts
|
|
10565
|
-
var AIWG_SCAN_REQUIRED_FIELDS = [
|
|
10566
|
-
"schema_version",
|
|
10567
|
-
"id",
|
|
10568
|
-
"type",
|
|
10569
|
-
"title",
|
|
10570
|
-
"text",
|
|
10571
|
-
"facets",
|
|
10572
|
-
"tags",
|
|
10573
|
-
"concepts",
|
|
10574
|
-
"privacy"
|
|
10575
|
-
];
|
|
10576
|
-
function isPrivacyExcluded(record, options) {
|
|
10577
|
-
const privacy = record.privacy;
|
|
10578
|
-
if (!privacy || !isPrivacyClassification(privacy.classification) || typeof privacy.pii !== "boolean") return true;
|
|
10579
|
-
if (privacy.classification === "private" && !options?.includePrivate) return true;
|
|
10580
|
-
if (privacy.pii && !options?.includePii) return true;
|
|
10581
|
-
return false;
|
|
10582
10582
|
}
|
|
10583
|
-
|
|
10584
|
-
|
|
10585
|
-
|
|
10586
|
-
|
|
10587
|
-
"
|
|
10588
|
-
"
|
|
10589
|
-
"
|
|
10590
|
-
"
|
|
10591
|
-
"facets",
|
|
10592
|
-
"tags",
|
|
10593
|
-
"concepts",
|
|
10594
|
-
"relationships",
|
|
10595
|
-
"provenance",
|
|
10596
|
-
"privacy",
|
|
10597
|
-
"updated_at"
|
|
10583
|
+
var V2_ONLY_RECORD_FIELDS = [
|
|
10584
|
+
"search",
|
|
10585
|
+
"chunks",
|
|
10586
|
+
"embeddings",
|
|
10587
|
+
"skos_concepts",
|
|
10588
|
+
"skos_relations",
|
|
10589
|
+
"provenance_events",
|
|
10590
|
+
"compatibility"
|
|
10598
10591
|
];
|
|
10599
|
-
var
|
|
10600
|
-
|
|
10601
|
-
|
|
10602
|
-
|
|
10603
|
-
|
|
10604
|
-
|
|
10605
|
-
|
|
10606
|
-
|
|
10607
|
-
|
|
10608
|
-
function hasString(value) {
|
|
10609
|
-
return typeof value === "string" && value.length > 0;
|
|
10610
|
-
}
|
|
10611
|
-
function pushFacet(counts, name, value) {
|
|
10612
|
-
let bucket = counts[name];
|
|
10613
|
-
if (bucket === void 0) {
|
|
10614
|
-
bucket = /* @__PURE__ */ Object.create(null);
|
|
10615
|
-
counts[name] = bucket;
|
|
10616
|
-
}
|
|
10617
|
-
bucket[value] = (bucket[value] ?? 0) + 1;
|
|
10618
|
-
}
|
|
10619
|
-
function hasNonNegativeInteger(value) {
|
|
10620
|
-
return Number.isInteger(value) && typeof value === "number" && value >= 0;
|
|
10621
|
-
}
|
|
10622
|
-
function hasPositiveInteger(value) {
|
|
10623
|
-
return Number.isInteger(value) && typeof value === "number" && value > 0;
|
|
10624
|
-
}
|
|
10625
|
-
function isFacetCounts(value) {
|
|
10626
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
10627
|
-
return Object.values(value).every((counts) => !!counts && typeof counts === "object" && !Array.isArray(counts) && Object.values(counts).every((count) => hasNonNegativeInteger(count)));
|
|
10628
|
-
}
|
|
10629
|
-
function isPlainRecord(value) {
|
|
10630
|
-
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
10631
|
-
}
|
|
10632
|
-
function isOptionalStringArray(value) {
|
|
10633
|
-
return value === void 0 || Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
10634
|
-
}
|
|
10635
|
-
function isSupportedIndexSchemaVersion(value) {
|
|
10636
|
-
return value === "aiwg.fortemi.index.export.v1" || value === "aiwg.fortemi.index.export.v2";
|
|
10637
|
-
}
|
|
10638
|
-
function isSupportedRecordSchemaVersion(value) {
|
|
10639
|
-
return value === "aiwg.fortemi.index.record.v1" || value === "aiwg.fortemi.index.record.v2";
|
|
10640
|
-
}
|
|
10641
|
-
function validateOptionalRichMetadata(item, index, errors) {
|
|
10642
|
-
if (item.skos_concepts !== void 0) {
|
|
10643
|
-
if (!Array.isArray(item.skos_concepts)) {
|
|
10644
|
-
errors.push("items[" + index + "].skos_concepts must be an array when present");
|
|
10645
|
-
} else {
|
|
10646
|
-
for (const [conceptIndex, concept] of item.skos_concepts.entries()) {
|
|
10647
|
-
if (!isPlainRecord(concept)) {
|
|
10648
|
-
errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "] must be an object");
|
|
10649
|
-
continue;
|
|
10650
|
-
}
|
|
10651
|
-
if (!hasString(concept.id)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].id is required");
|
|
10652
|
-
if (!hasString(concept.prefLabel)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].prefLabel is required");
|
|
10653
|
-
if (!isOptionalStringArray(concept.altLabels)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].altLabels must be a string array");
|
|
10654
|
-
if (concept.metadata !== void 0 && !isPlainRecord(concept.metadata)) {
|
|
10655
|
-
errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].metadata must be an object");
|
|
10656
|
-
}
|
|
10657
|
-
}
|
|
10658
|
-
}
|
|
10592
|
+
var V2_ONLY_SOURCE_FIELDS = ["origin", "generated", "checksum", "updated_at"];
|
|
10593
|
+
var V2_ONLY_RELATIONSHIP_FIELDS = ["target_path", "direction", "metadata"];
|
|
10594
|
+
function forbidV2FieldsOnV1Record(item, index, errors) {
|
|
10595
|
+
if (item.schema_version !== "aiwg.fortemi.index.record.v1") return;
|
|
10596
|
+
const at = "items[" + index + "]";
|
|
10597
|
+
const bag = item;
|
|
10598
|
+
const v2msg = " is a v2-only field and must be absent on a record.v1 record";
|
|
10599
|
+
for (const field of V2_ONLY_RECORD_FIELDS) {
|
|
10600
|
+
if (bag[field] !== void 0) errors.push(at + "." + field + v2msg);
|
|
10659
10601
|
}
|
|
10660
|
-
if (item.
|
|
10661
|
-
|
|
10662
|
-
|
|
10663
|
-
|
|
10664
|
-
for (const [relationIndex, relation] of item.skos_relations.entries()) {
|
|
10665
|
-
if (!isPlainRecord(relation)) {
|
|
10666
|
-
errors.push("items[" + index + "].skos_relations[" + relationIndex + "] must be an object");
|
|
10667
|
-
continue;
|
|
10668
|
-
}
|
|
10669
|
-
if (!hasString(relation.type)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].type is required");
|
|
10670
|
-
if (!hasString(relation.source_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].source_id is required");
|
|
10671
|
-
if (!hasString(relation.target_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].target_id is required");
|
|
10672
|
-
if (relation.metadata !== void 0 && !isPlainRecord(relation.metadata)) {
|
|
10673
|
-
errors.push("items[" + index + "].skos_relations[" + relationIndex + "].metadata must be an object");
|
|
10674
|
-
}
|
|
10675
|
-
}
|
|
10602
|
+
if (isPlainRecord(item.source)) {
|
|
10603
|
+
const src = item.source;
|
|
10604
|
+
for (const field of V2_ONLY_SOURCE_FIELDS) {
|
|
10605
|
+
if (src[field] !== void 0) errors.push(at + ".source." + field + v2msg);
|
|
10676
10606
|
}
|
|
10677
10607
|
}
|
|
10678
|
-
if (item.
|
|
10679
|
-
|
|
10680
|
-
errors.push("items[" + index + "].provenance_events must be an array when present");
|
|
10681
|
-
} else {
|
|
10682
|
-
for (const [eventIndex, event] of item.provenance_events.entries()) {
|
|
10683
|
-
if (!isPlainRecord(event)) {
|
|
10684
|
-
errors.push("items[" + index + "].provenance_events[" + eventIndex + "] must be an object");
|
|
10685
|
-
continue;
|
|
10686
|
-
}
|
|
10687
|
-
if (!hasString(event.activity)) errors.push("items[" + index + "].provenance_events[" + eventIndex + "].activity is required");
|
|
10688
|
-
if (event.attributes !== void 0 && !isPlainRecord(event.attributes)) {
|
|
10689
|
-
errors.push("items[" + index + "].provenance_events[" + eventIndex + "].attributes must be an object");
|
|
10690
|
-
}
|
|
10691
|
-
}
|
|
10692
|
-
}
|
|
10608
|
+
if (isPlainRecord(item.privacy) && item.privacy.locality !== void 0) {
|
|
10609
|
+
errors.push(at + ".privacy.locality" + v2msg);
|
|
10693
10610
|
}
|
|
10694
10611
|
if (Array.isArray(item.relationships)) {
|
|
10695
|
-
for (const [
|
|
10696
|
-
if (!isPlainRecord(
|
|
10697
|
-
|
|
10698
|
-
|
|
10699
|
-
|
|
10700
|
-
if (relationship.metadata !== void 0 && !isPlainRecord(relationship.metadata)) {
|
|
10701
|
-
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].metadata must be an object");
|
|
10702
|
-
}
|
|
10703
|
-
if (relationship.direction !== void 0 && relationship.direction !== "upstream" && relationship.direction !== "downstream" && relationship.direction !== "related") {
|
|
10704
|
-
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].direction must be upstream, downstream, or related");
|
|
10705
|
-
}
|
|
10706
|
-
if (relationship.target_path !== void 0 && typeof relationship.target_path !== "string") {
|
|
10707
|
-
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].target_path must be a string");
|
|
10708
|
-
}
|
|
10709
|
-
}
|
|
10710
|
-
}
|
|
10711
|
-
if (item.search !== void 0) {
|
|
10712
|
-
if (!isPlainRecord(item.search)) {
|
|
10713
|
-
errors.push("items[" + index + "].search must be an object");
|
|
10714
|
-
} else {
|
|
10715
|
-
if (!isOptionalStringArray(item.search.triggers)) errors.push("items[" + index + "].search.triggers must be a string array");
|
|
10716
|
-
if (!isOptionalStringArray(item.search.aliases)) errors.push("items[" + index + "].search.aliases must be a string array");
|
|
10717
|
-
if (!isOptionalStringArray(item.search.tags)) errors.push("items[" + index + "].search.tags must be a string array");
|
|
10718
|
-
if (item.search.frontmatter !== void 0 && !isPlainRecord(item.search.frontmatter)) {
|
|
10719
|
-
errors.push("items[" + index + "].search.frontmatter must be an object");
|
|
10720
|
-
}
|
|
10721
|
-
}
|
|
10722
|
-
}
|
|
10723
|
-
if (item.chunks !== void 0) {
|
|
10724
|
-
if (!Array.isArray(item.chunks)) {
|
|
10725
|
-
errors.push("items[" + index + "].chunks must be an array when present");
|
|
10726
|
-
} else {
|
|
10727
|
-
for (const [chunkIndex, chunk] of item.chunks.entries()) {
|
|
10728
|
-
if (!isPlainRecord(chunk)) {
|
|
10729
|
-
errors.push("items[" + index + "].chunks[" + chunkIndex + "] must be an object");
|
|
10730
|
-
continue;
|
|
10731
|
-
}
|
|
10732
|
-
if (chunk.metadata !== void 0 && !isPlainRecord(chunk.metadata)) {
|
|
10733
|
-
errors.push("items[" + index + "].chunks[" + chunkIndex + "].metadata must be an object");
|
|
10734
|
-
}
|
|
10735
|
-
}
|
|
10736
|
-
}
|
|
10737
|
-
}
|
|
10738
|
-
if (item.embeddings !== void 0) {
|
|
10739
|
-
if (!Array.isArray(item.embeddings)) {
|
|
10740
|
-
errors.push("items[" + index + "].embeddings must be an array when present");
|
|
10741
|
-
} else {
|
|
10742
|
-
for (const [embeddingIndex, embedding] of item.embeddings.entries()) {
|
|
10743
|
-
if (!isPlainRecord(embedding)) {
|
|
10744
|
-
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "] must be an object");
|
|
10745
|
-
continue;
|
|
10746
|
-
}
|
|
10747
|
-
const vector = embedding.embedding ?? embedding.vector;
|
|
10748
|
-
if (vector !== void 0 && (!Array.isArray(vector) || !vector.every((entry) => typeof entry === "number"))) {
|
|
10749
|
-
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "].embedding/vector must be a number array");
|
|
10750
|
-
}
|
|
10751
|
-
if (embedding.metadata !== void 0 && !isPlainRecord(embedding.metadata)) {
|
|
10752
|
-
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "].metadata must be an object");
|
|
10753
|
-
}
|
|
10754
|
-
}
|
|
10755
|
-
}
|
|
10756
|
-
}
|
|
10757
|
-
if (item.compatibility !== void 0 && !isPlainRecord(item.compatibility)) {
|
|
10758
|
-
errors.push("items[" + index + "].compatibility must be an object");
|
|
10759
|
-
}
|
|
10760
|
-
}
|
|
10761
|
-
function isPrivacyClassification(value) {
|
|
10762
|
-
return value === "private" || value === "sanitized" || value === "public";
|
|
10763
|
-
}
|
|
10764
|
-
function isProvenanceConfidence(value) {
|
|
10765
|
-
return value === "source" || value === "candidate" || value === "reviewed" || value === "rejected";
|
|
10766
|
-
}
|
|
10767
|
-
function validateProvenanceItems(item, index, errors) {
|
|
10768
|
-
if (!Array.isArray(item.provenance)) return;
|
|
10769
|
-
for (const [provIndex, prov] of item.provenance.entries()) {
|
|
10770
|
-
const at = "items[" + index + "].provenance[" + provIndex + "]";
|
|
10771
|
-
if (!isPlainRecord(prov)) {
|
|
10772
|
-
errors.push(at + " must be an object");
|
|
10773
|
-
continue;
|
|
10774
|
-
}
|
|
10775
|
-
if (!hasString(prov.field)) errors.push(at + ".field is required");
|
|
10776
|
-
if (!hasString(prov.source)) errors.push(at + ".source is required");
|
|
10777
|
-
if (!hasString(prov.path)) errors.push(at + ".path is required");
|
|
10778
|
-
if (!isProvenanceConfidence(prov.confidence)) errors.push(at + ".confidence must be one of source, candidate, reviewed, rejected");
|
|
10779
|
-
if (!isPrivacyClassification(prov.privacy)) errors.push(at + ".privacy must be one of private, sanitized, public");
|
|
10780
|
-
}
|
|
10781
|
-
}
|
|
10782
|
-
var V2_ONLY_RECORD_FIELDS = ["search", "chunks", "embeddings", "skos_concepts", "skos_relations", "compatibility"];
|
|
10783
|
-
var V2_ONLY_SOURCE_FIELDS = ["origin", "generated", "checksum", "updated_at"];
|
|
10784
|
-
var V2_ONLY_RELATIONSHIP_FIELDS = ["target_path", "direction", "metadata"];
|
|
10785
|
-
function forbidV2FieldsOnV1Record(item, index, errors) {
|
|
10786
|
-
if (item.schema_version !== "aiwg.fortemi.index.record.v1") return;
|
|
10787
|
-
const at = "items[" + index + "]";
|
|
10788
|
-
const bag = item;
|
|
10789
|
-
const v2msg = " is a v2-only field and must be absent on a record.v1 record";
|
|
10790
|
-
for (const field of V2_ONLY_RECORD_FIELDS) {
|
|
10791
|
-
if (bag[field] !== void 0) errors.push(at + "." + field + v2msg);
|
|
10792
|
-
}
|
|
10793
|
-
if (isPlainRecord(item.source)) {
|
|
10794
|
-
const src = item.source;
|
|
10795
|
-
for (const field of V2_ONLY_SOURCE_FIELDS) {
|
|
10796
|
-
if (src[field] !== void 0) errors.push(at + ".source." + field + v2msg);
|
|
10797
|
-
}
|
|
10798
|
-
}
|
|
10799
|
-
if (isPlainRecord(item.privacy) && item.privacy.locality !== void 0) {
|
|
10800
|
-
errors.push(at + ".privacy.locality" + v2msg);
|
|
10801
|
-
}
|
|
10802
|
-
if (Array.isArray(item.relationships)) {
|
|
10803
|
-
for (const [relIndex, rel] of item.relationships.entries()) {
|
|
10804
|
-
if (!isPlainRecord(rel)) continue;
|
|
10805
|
-
const relBag = rel;
|
|
10806
|
-
for (const field of V2_ONLY_RELATIONSHIP_FIELDS) {
|
|
10807
|
-
if (relBag[field] !== void 0) errors.push(at + ".relationships[" + relIndex + "]." + field + v2msg);
|
|
10612
|
+
for (const [relIndex, rel] of item.relationships.entries()) {
|
|
10613
|
+
if (!isPlainRecord(rel)) continue;
|
|
10614
|
+
const relBag = rel;
|
|
10615
|
+
for (const field of V2_ONLY_RELATIONSHIP_FIELDS) {
|
|
10616
|
+
if (relBag[field] !== void 0) errors.push(at + ".relationships[" + relIndex + "]." + field + v2msg);
|
|
10808
10617
|
}
|
|
10809
10618
|
}
|
|
10810
10619
|
}
|
|
10811
10620
|
}
|
|
10812
10621
|
function validateAiwgFortemiIndexExport(value) {
|
|
10813
|
-
const errors =
|
|
10622
|
+
const errors = [];
|
|
10814
10623
|
const counts = /* @__PURE__ */ Object.create(null);
|
|
10815
10624
|
const data = isPlainRecord(value) ? value : {};
|
|
10816
10625
|
if (!isPlainRecord(value)) errors.push("index export must be an object");
|
|
@@ -10849,6 +10658,9 @@ function validateAiwgFortemiIndexExport(value) {
|
|
|
10849
10658
|
if (!isSupportedRecordSchemaVersion(item.schema_version)) {
|
|
10850
10659
|
errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1 or aiwg.fortemi.index.record.v2");
|
|
10851
10660
|
}
|
|
10661
|
+
if (data.schema_version === "aiwg.fortemi.index.export.v1" && item.schema_version === "aiwg.fortemi.index.record.v2") {
|
|
10662
|
+
errors.push("items[" + index + "].schema_version must match aiwg.fortemi.index.export.v1");
|
|
10663
|
+
}
|
|
10852
10664
|
if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
|
|
10853
10665
|
if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
|
|
10854
10666
|
if (hasString(item.id)) ids.add(item.id);
|
|
@@ -10956,6 +10768,33 @@ function assertAiwgFortemiChunkManifest(value) {
|
|
|
10956
10768
|
}
|
|
10957
10769
|
return value;
|
|
10958
10770
|
}
|
|
10771
|
+
function validateProjectedRecordShape(item, index, errors) {
|
|
10772
|
+
const at = "items[" + index + "]";
|
|
10773
|
+
if (!item.privacy || !isPlainRecord(item.privacy)) {
|
|
10774
|
+
errors.push(at + "/privacy requires classification and pii");
|
|
10775
|
+
} else {
|
|
10776
|
+
if (!isPrivacyClassification(item.privacy.classification)) {
|
|
10777
|
+
errors.push(at + "/privacy/classification must be one of private, sanitized, public");
|
|
10778
|
+
}
|
|
10779
|
+
if (typeof item.privacy.pii !== "boolean") {
|
|
10780
|
+
errors.push(at + "/privacy/pii must be boolean");
|
|
10781
|
+
}
|
|
10782
|
+
}
|
|
10783
|
+
if (Array.isArray(item.provenance)) {
|
|
10784
|
+
for (const [provIndex, prov] of item.provenance.entries()) {
|
|
10785
|
+
if (!isPlainRecord(prov)) {
|
|
10786
|
+
errors.push(at + "/provenance/" + provIndex + " must be an object");
|
|
10787
|
+
continue;
|
|
10788
|
+
}
|
|
10789
|
+
if (!isProvenanceConfidence(prov.confidence)) {
|
|
10790
|
+
errors.push(at + "/provenance/" + provIndex + "/confidence must be one of source, candidate, reviewed, rejected");
|
|
10791
|
+
}
|
|
10792
|
+
if (!isPrivacyClassification(prov.privacy)) {
|
|
10793
|
+
errors.push(at + "/provenance/" + provIndex + "/privacy must be one of private, sanitized, public");
|
|
10794
|
+
}
|
|
10795
|
+
}
|
|
10796
|
+
}
|
|
10797
|
+
}
|
|
10959
10798
|
function validateProjectedRecords(items, sourceSchemaVersion) {
|
|
10960
10799
|
const errors = [];
|
|
10961
10800
|
const ids = /* @__PURE__ */ new Set();
|
|
@@ -10965,8 +10804,7 @@ function validateProjectedRecords(items, sourceSchemaVersion) {
|
|
|
10965
10804
|
errors.push("items[" + index + "] must be an object");
|
|
10966
10805
|
continue;
|
|
10967
10806
|
}
|
|
10968
|
-
|
|
10969
|
-
errors.push(...schemaValidation.errors.map((error) => `items[${index}]${error}`));
|
|
10807
|
+
validateProjectedRecordShape(item, index, errors);
|
|
10970
10808
|
const expectedRecordVersion = sourceSchemaVersion === "aiwg.fortemi.index.export.v2" ? "aiwg.fortemi.index.record.v2" : "aiwg.fortemi.index.record.v1";
|
|
10971
10809
|
if (item.schema_version !== expectedRecordVersion) {
|
|
10972
10810
|
errors.push(`items[${index}].schema_version must match ${sourceSchemaVersion}`);
|
|
@@ -11130,6 +10968,137 @@ function generatedAtString(value) {
|
|
|
11130
10968
|
if (value instanceof Date) return value.toISOString();
|
|
11131
10969
|
return value ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
11132
10970
|
}
|
|
10971
|
+
var SHA256_K = [
|
|
10972
|
+
1116352408,
|
|
10973
|
+
1899447441,
|
|
10974
|
+
3049323471,
|
|
10975
|
+
3921009573,
|
|
10976
|
+
961987163,
|
|
10977
|
+
1508970993,
|
|
10978
|
+
2453635748,
|
|
10979
|
+
2870763221,
|
|
10980
|
+
3624381080,
|
|
10981
|
+
310598401,
|
|
10982
|
+
607225278,
|
|
10983
|
+
1426881987,
|
|
10984
|
+
1925078388,
|
|
10985
|
+
2162078206,
|
|
10986
|
+
2614888103,
|
|
10987
|
+
3248222580,
|
|
10988
|
+
3835390401,
|
|
10989
|
+
4022224774,
|
|
10990
|
+
264347078,
|
|
10991
|
+
604807628,
|
|
10992
|
+
770255983,
|
|
10993
|
+
1249150122,
|
|
10994
|
+
1555081692,
|
|
10995
|
+
1996064986,
|
|
10996
|
+
2554220882,
|
|
10997
|
+
2821834349,
|
|
10998
|
+
2952996808,
|
|
10999
|
+
3210313671,
|
|
11000
|
+
3336571891,
|
|
11001
|
+
3584528711,
|
|
11002
|
+
113926993,
|
|
11003
|
+
338241895,
|
|
11004
|
+
666307205,
|
|
11005
|
+
773529912,
|
|
11006
|
+
1294757372,
|
|
11007
|
+
1396182291,
|
|
11008
|
+
1695183700,
|
|
11009
|
+
1986661051,
|
|
11010
|
+
2177026350,
|
|
11011
|
+
2456956037,
|
|
11012
|
+
2730485921,
|
|
11013
|
+
2820302411,
|
|
11014
|
+
3259730800,
|
|
11015
|
+
3345764771,
|
|
11016
|
+
3516065817,
|
|
11017
|
+
3600352804,
|
|
11018
|
+
4094571909,
|
|
11019
|
+
275423344,
|
|
11020
|
+
430227734,
|
|
11021
|
+
506948616,
|
|
11022
|
+
659060556,
|
|
11023
|
+
883997877,
|
|
11024
|
+
958139571,
|
|
11025
|
+
1322822218,
|
|
11026
|
+
1537002063,
|
|
11027
|
+
1747873779,
|
|
11028
|
+
1955562222,
|
|
11029
|
+
2024104815,
|
|
11030
|
+
2227730452,
|
|
11031
|
+
2361852424,
|
|
11032
|
+
2428436474,
|
|
11033
|
+
2756734187,
|
|
11034
|
+
3204031479,
|
|
11035
|
+
3329325298
|
|
11036
|
+
];
|
|
11037
|
+
function rotr(value, bits) {
|
|
11038
|
+
return value >>> bits | value << 32 - bits;
|
|
11039
|
+
}
|
|
11040
|
+
function sha256Hex2(data) {
|
|
11041
|
+
const bitLength = data.length * 8;
|
|
11042
|
+
const padded = new Uint8Array(data.length + 9 + 63 >> 6 << 6);
|
|
11043
|
+
padded.set(data);
|
|
11044
|
+
padded[data.length] = 128;
|
|
11045
|
+
const view = new DataView(padded.buffer);
|
|
11046
|
+
view.setUint32(padded.length - 4, bitLength >>> 0);
|
|
11047
|
+
view.setUint32(padded.length - 8, Math.floor(bitLength / 4294967296));
|
|
11048
|
+
let h0 = 1779033703;
|
|
11049
|
+
let h1 = 3144134277;
|
|
11050
|
+
let h2 = 1013904242;
|
|
11051
|
+
let h3 = 2773480762;
|
|
11052
|
+
let h4 = 1359893119;
|
|
11053
|
+
let h5 = 2600822924;
|
|
11054
|
+
let h6 = 528734635;
|
|
11055
|
+
let h7 = 1541459225;
|
|
11056
|
+
const w = new Uint32Array(64);
|
|
11057
|
+
for (let offset = 0; offset < padded.length; offset += 64) {
|
|
11058
|
+
for (let i = 0; i < 16; i++) w[i] = view.getUint32(offset + i * 4);
|
|
11059
|
+
for (let i = 16; i < 64; i++) {
|
|
11060
|
+
const s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ w[i - 15] >>> 3;
|
|
11061
|
+
const s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ w[i - 2] >>> 10;
|
|
11062
|
+
w[i] = w[i - 16] + s0 + w[i - 7] + s1 >>> 0;
|
|
11063
|
+
}
|
|
11064
|
+
let a = h0;
|
|
11065
|
+
let b = h1;
|
|
11066
|
+
let c = h2;
|
|
11067
|
+
let d = h3;
|
|
11068
|
+
let e = h4;
|
|
11069
|
+
let f = h5;
|
|
11070
|
+
let g = h6;
|
|
11071
|
+
let h = h7;
|
|
11072
|
+
for (let i = 0; i < 64; i++) {
|
|
11073
|
+
const s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
|
|
11074
|
+
const ch = e & f ^ ~e & g;
|
|
11075
|
+
const temp1 = h + s1 + ch + SHA256_K[i] + w[i] >>> 0;
|
|
11076
|
+
const s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
|
|
11077
|
+
const maj = a & b ^ a & c ^ b & c;
|
|
11078
|
+
const temp2 = s0 + maj >>> 0;
|
|
11079
|
+
h = g;
|
|
11080
|
+
g = f;
|
|
11081
|
+
f = e;
|
|
11082
|
+
e = d + temp1 >>> 0;
|
|
11083
|
+
d = c;
|
|
11084
|
+
c = b;
|
|
11085
|
+
b = a;
|
|
11086
|
+
a = temp1 + temp2 >>> 0;
|
|
11087
|
+
}
|
|
11088
|
+
h0 = h0 + a >>> 0;
|
|
11089
|
+
h1 = h1 + b >>> 0;
|
|
11090
|
+
h2 = h2 + c >>> 0;
|
|
11091
|
+
h3 = h3 + d >>> 0;
|
|
11092
|
+
h4 = h4 + e >>> 0;
|
|
11093
|
+
h5 = h5 + f >>> 0;
|
|
11094
|
+
h6 = h6 + g >>> 0;
|
|
11095
|
+
h7 = h7 + h >>> 0;
|
|
11096
|
+
}
|
|
11097
|
+
return [h0, h1, h2, h3, h4, h5, h6, h7].map((part) => part.toString(16).padStart(8, "0")).join("");
|
|
11098
|
+
}
|
|
11099
|
+
function computeAiwgIndexHash(data) {
|
|
11100
|
+
return `sha256:${sha256Hex2(data)}`;
|
|
11101
|
+
}
|
|
11133
11102
|
async function buildAiwgStaticEmbeddingSet(index, options) {
|
|
11134
11103
|
assertAiwgFortemiIndexExport(index);
|
|
11135
11104
|
const granularity = options.granularity ?? "body";
|
|
@@ -11145,7 +11114,7 @@ async function buildAiwgStaticEmbeddingSet(index, options) {
|
|
|
11145
11114
|
record_id: record.id,
|
|
11146
11115
|
embedding,
|
|
11147
11116
|
granularity,
|
|
11148
|
-
input_hash:
|
|
11117
|
+
input_hash: computeAiwgIndexHash(new TextEncoder().encode(input)),
|
|
11149
11118
|
source_path: record.source.path
|
|
11150
11119
|
});
|
|
11151
11120
|
}
|
|
@@ -11891,327 +11860,1867 @@ async function recordsFromChunkedRuntime(runtime, onProgress) {
|
|
|
11891
11860
|
records.push(item.relationships ? item : await getChunkRecord(runtime, item.id));
|
|
11892
11861
|
}
|
|
11893
11862
|
}
|
|
11894
|
-
return { records, scannedParts, fetchedParts };
|
|
11895
|
-
}
|
|
11896
|
-
async function relationshipResultFromChunkedRuntime(runtime, options = {}) {
|
|
11897
|
-
const loaded = await recordsFromChunkedRuntime(runtime);
|
|
11898
|
-
return {
|
|
11899
|
-
...relationshipResultFromRecords(loaded.records, options),
|
|
11900
|
-
scannedParts: loaded.scannedParts,
|
|
11901
|
-
fetchedParts: loaded.fetchedParts
|
|
11902
|
-
};
|
|
11863
|
+
return { records, scannedParts, fetchedParts };
|
|
11864
|
+
}
|
|
11865
|
+
async function relationshipResultFromChunkedRuntime(runtime, options = {}) {
|
|
11866
|
+
const loaded = await recordsFromChunkedRuntime(runtime);
|
|
11867
|
+
return {
|
|
11868
|
+
...relationshipResultFromRecords(loaded.records, options),
|
|
11869
|
+
scannedParts: loaded.scannedParts,
|
|
11870
|
+
fetchedParts: loaded.fetchedParts
|
|
11871
|
+
};
|
|
11872
|
+
}
|
|
11873
|
+
async function queryChunkedAiwgFortemiIndex(runtime, query = "", options = {}) {
|
|
11874
|
+
const q = query.trim().toLowerCase();
|
|
11875
|
+
let scannedParts = 0;
|
|
11876
|
+
let fetchedParts = 0;
|
|
11877
|
+
if (isDirectChunkBrowse(query, options)) {
|
|
11878
|
+
const offset = options.offset ?? 0;
|
|
11879
|
+
const limit = options.limit ?? runtime.manifest.total;
|
|
11880
|
+
const parts = getPartsForRange(runtime.manifest, offset, limit);
|
|
11881
|
+
const items = [];
|
|
11882
|
+
for (const partRef of parts) {
|
|
11883
|
+
const loaded = await loadChunkPart(runtime, partRef);
|
|
11884
|
+
if (loaded.fetched) fetchedParts += 1;
|
|
11885
|
+
scannedParts += 1;
|
|
11886
|
+
options.onProgress?.({ phase: "part", done: scannedParts, total: parts.length, href: partRef.href });
|
|
11887
|
+
const start = Math.max(0, offset - partRef.offset);
|
|
11888
|
+
const end = Math.min(loaded.part.items.length, offset + limit - partRef.offset);
|
|
11889
|
+
items.push(...loaded.part.items.slice(start, end));
|
|
11890
|
+
}
|
|
11891
|
+
return {
|
|
11892
|
+
items,
|
|
11893
|
+
total: runtime.manifest.total,
|
|
11894
|
+
facets: runtime.manifest.facets ?? {},
|
|
11895
|
+
manifestTotal: runtime.manifest.total,
|
|
11896
|
+
scannedParts,
|
|
11897
|
+
fetchedParts,
|
|
11898
|
+
complete: true
|
|
11899
|
+
};
|
|
11900
|
+
}
|
|
11901
|
+
const matchKey = matchSetCacheKey(q, options);
|
|
11902
|
+
const cached = runtime.matchCache.get(matchKey);
|
|
11903
|
+
if (cached) {
|
|
11904
|
+
runtime.matchCache.delete(matchKey);
|
|
11905
|
+
runtime.matchCache.set(matchKey, cached);
|
|
11906
|
+
return {
|
|
11907
|
+
...createQueryResultFromRankedEntries(cached, q, options),
|
|
11908
|
+
manifestTotal: runtime.manifest.total,
|
|
11909
|
+
scannedParts: 0,
|
|
11910
|
+
fetchedParts: 0,
|
|
11911
|
+
complete: true
|
|
11912
|
+
};
|
|
11913
|
+
}
|
|
11914
|
+
const entries = [];
|
|
11915
|
+
for (const partRef of runtime.manifest.parts) {
|
|
11916
|
+
const loaded = await loadChunkPart(runtime, partRef);
|
|
11917
|
+
if (loaded.fetched) fetchedParts += 1;
|
|
11918
|
+
scannedParts += 1;
|
|
11919
|
+
options.onProgress?.({ phase: "part", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
|
|
11920
|
+
entries.push(...createRankedEntries(loaded.part.items, q, options, partRef.offset));
|
|
11921
|
+
options.onProgress?.({ phase: "query", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
|
|
11922
|
+
}
|
|
11923
|
+
cacheMatchEntries(runtime, matchKey, entries);
|
|
11924
|
+
return {
|
|
11925
|
+
...createQueryResultFromRankedEntries(entries, q, options),
|
|
11926
|
+
manifestTotal: runtime.manifest.total,
|
|
11927
|
+
scannedParts,
|
|
11928
|
+
fetchedParts,
|
|
11929
|
+
complete: true
|
|
11930
|
+
};
|
|
11931
|
+
}
|
|
11932
|
+
function createAiwgReviewDecisionExport(source, decisions, generatedAt = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
11933
|
+
return {
|
|
11934
|
+
schema_version: "aiwg.fortemi.review-decisions.v1",
|
|
11935
|
+
generated_at: generatedAt,
|
|
11936
|
+
source_export_schema_version: source.schema_version,
|
|
11937
|
+
decisions: [...decisions].sort((left, right) => left.item_id.localeCompare(right.item_id))
|
|
11938
|
+
};
|
|
11939
|
+
}
|
|
11940
|
+
function createAiwgIndexController(initialIndex) {
|
|
11941
|
+
let index = initialIndex ?? null;
|
|
11942
|
+
let chunked = null;
|
|
11943
|
+
let data = null;
|
|
11944
|
+
let error = null;
|
|
11945
|
+
let reviewDecisions = [];
|
|
11946
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
11947
|
+
const snapshot = () => ({
|
|
11948
|
+
index,
|
|
11949
|
+
chunked: chunked ? {
|
|
11950
|
+
manifest: chunked.manifest,
|
|
11951
|
+
cachedParts: chunked.partCache.size,
|
|
11952
|
+
maxCachedParts: chunked.maxCachedParts
|
|
11953
|
+
} : null,
|
|
11954
|
+
data,
|
|
11955
|
+
error,
|
|
11956
|
+
reviewDecisions: [...reviewDecisions]
|
|
11957
|
+
});
|
|
11958
|
+
const notify = () => {
|
|
11959
|
+
const current = snapshot();
|
|
11960
|
+
for (const listener of listeners) listener(current);
|
|
11961
|
+
};
|
|
11962
|
+
const requireIndex = () => {
|
|
11963
|
+
if (!index) throw new Error("No AIWG index export loaded");
|
|
11964
|
+
return index;
|
|
11965
|
+
};
|
|
11966
|
+
return {
|
|
11967
|
+
loadIndex(value) {
|
|
11968
|
+
try {
|
|
11969
|
+
const parsed = assertAiwgFortemiIndexExport(value);
|
|
11970
|
+
index = parsed;
|
|
11971
|
+
chunked = null;
|
|
11972
|
+
data = null;
|
|
11973
|
+
reviewDecisions = [];
|
|
11974
|
+
error = null;
|
|
11975
|
+
notify();
|
|
11976
|
+
return parsed;
|
|
11977
|
+
} catch (err) {
|
|
11978
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
11979
|
+
notify();
|
|
11980
|
+
throw error;
|
|
11981
|
+
}
|
|
11982
|
+
},
|
|
11983
|
+
loadChunkedIndex(manifest, loader, options = {}) {
|
|
11984
|
+
try {
|
|
11985
|
+
const parsed = assertAiwgFortemiChunkManifest(manifest);
|
|
11986
|
+
index = null;
|
|
11987
|
+
chunked = {
|
|
11988
|
+
manifest: parsed,
|
|
11989
|
+
loader,
|
|
11990
|
+
maxCachedParts: clampMaxCachedParts(options.maxCachedParts),
|
|
11991
|
+
partCache: /* @__PURE__ */ new Map(),
|
|
11992
|
+
detailLoader: options.detailLoader,
|
|
11993
|
+
maxCachedDetails: clampMaxCachedDetails(options.maxCachedDetails),
|
|
11994
|
+
detailCache: /* @__PURE__ */ new Map(),
|
|
11995
|
+
maxCachedMatches: clampMaxCachedMatches(options.maxCachedMatches),
|
|
11996
|
+
matchCache: /* @__PURE__ */ new Map()
|
|
11997
|
+
};
|
|
11998
|
+
data = null;
|
|
11999
|
+
reviewDecisions = [];
|
|
12000
|
+
error = null;
|
|
12001
|
+
notify();
|
|
12002
|
+
return parsed;
|
|
12003
|
+
} catch (err) {
|
|
12004
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
12005
|
+
notify();
|
|
12006
|
+
throw error;
|
|
12007
|
+
}
|
|
12008
|
+
},
|
|
12009
|
+
getIndex() {
|
|
12010
|
+
return index;
|
|
12011
|
+
},
|
|
12012
|
+
getChunkedManifest() {
|
|
12013
|
+
return chunked?.manifest ?? null;
|
|
12014
|
+
},
|
|
12015
|
+
getSnapshot() {
|
|
12016
|
+
return snapshot();
|
|
12017
|
+
},
|
|
12018
|
+
query(query = "", options) {
|
|
12019
|
+
const result = queryAiwgFortemiIndex(requireIndex(), query, options);
|
|
12020
|
+
data = result;
|
|
12021
|
+
error = null;
|
|
12022
|
+
notify();
|
|
12023
|
+
return result;
|
|
12024
|
+
},
|
|
12025
|
+
async queryChunked(query = "", options) {
|
|
12026
|
+
if (!chunked) throw new Error("No AIWG chunked index manifest loaded");
|
|
12027
|
+
try {
|
|
12028
|
+
const result = await queryChunkedAiwgFortemiIndex(chunked, query, options);
|
|
12029
|
+
data = result;
|
|
12030
|
+
error = null;
|
|
12031
|
+
notify();
|
|
12032
|
+
return result;
|
|
12033
|
+
} catch (err) {
|
|
12034
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
12035
|
+
notify();
|
|
12036
|
+
throw error;
|
|
12037
|
+
}
|
|
12038
|
+
},
|
|
12039
|
+
async getRecord(id) {
|
|
12040
|
+
if (chunked) {
|
|
12041
|
+
try {
|
|
12042
|
+
return await getChunkRecord(chunked, id);
|
|
12043
|
+
} catch (err) {
|
|
12044
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
12045
|
+
notify();
|
|
12046
|
+
throw error;
|
|
12047
|
+
}
|
|
12048
|
+
}
|
|
12049
|
+
const found = requireIndex().items.find((item) => item.id === id);
|
|
12050
|
+
if (!found) throw new Error("Record not found: " + id);
|
|
12051
|
+
return found;
|
|
12052
|
+
},
|
|
12053
|
+
async neighbors(id, options) {
|
|
12054
|
+
try {
|
|
12055
|
+
const queryOptions = neighborQueryOptions(id, options);
|
|
12056
|
+
const result = chunked ? await relationshipResultFromChunkedRuntime(chunked, queryOptions) : relationshipResultFromRecords(requireIndex().items, queryOptions);
|
|
12057
|
+
return filterNeighborResult(id, result, options);
|
|
12058
|
+
} catch (err) {
|
|
12059
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
12060
|
+
notify();
|
|
12061
|
+
throw error;
|
|
12062
|
+
}
|
|
12063
|
+
},
|
|
12064
|
+
async relationshipQuery(options) {
|
|
12065
|
+
try {
|
|
12066
|
+
return chunked ? await relationshipResultFromChunkedRuntime(chunked, options) : relationshipResultFromRecords(requireIndex().items, options);
|
|
12067
|
+
} catch (err) {
|
|
12068
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
12069
|
+
notify();
|
|
12070
|
+
throw error;
|
|
12071
|
+
}
|
|
12072
|
+
},
|
|
12073
|
+
async relationshipSet(options) {
|
|
12074
|
+
const [left, right] = await Promise.all([
|
|
12075
|
+
this.neighbors(options.a, options),
|
|
12076
|
+
this.neighbors(options.b, options)
|
|
12077
|
+
]);
|
|
12078
|
+
const leftIds = new Set(left.nodes.map((node) => node.id).filter((id) => id !== options.a));
|
|
12079
|
+
const rightIds = new Set(right.nodes.map((node) => node.id).filter((id) => id !== options.b));
|
|
12080
|
+
let ids;
|
|
12081
|
+
if (options.op === "intersection") ids = [...leftIds].filter((id) => rightIds.has(id));
|
|
12082
|
+
else if (options.op === "difference") ids = [...leftIds].filter((id) => !rightIds.has(id));
|
|
12083
|
+
else ids = [.../* @__PURE__ */ new Set([...leftIds, ...rightIds])];
|
|
12084
|
+
return { op: options.op, ids: ids.sort() };
|
|
12085
|
+
},
|
|
12086
|
+
clearChunkCache() {
|
|
12087
|
+
chunked?.partCache.clear();
|
|
12088
|
+
chunked?.detailCache.clear();
|
|
12089
|
+
chunked?.matchCache.clear();
|
|
12090
|
+
error = null;
|
|
12091
|
+
notify();
|
|
12092
|
+
},
|
|
12093
|
+
toCommunityGraph(options) {
|
|
12094
|
+
return aiwgFortemiIndexToCommunityGraph(requireIndex(), options);
|
|
12095
|
+
},
|
|
12096
|
+
async toCommunityGraphChunked(options) {
|
|
12097
|
+
if (!chunked) return aiwgFortemiIndexToCommunityGraph(requireIndex(), options);
|
|
12098
|
+
const loaded = await recordsFromChunkedRuntime(chunked, options?.onProgress);
|
|
12099
|
+
return aiwgFortemiIndexToCommunityGraph({
|
|
12100
|
+
generated_at: chunked.manifest.generated_at,
|
|
12101
|
+
source: chunked.manifest.source,
|
|
12102
|
+
items: loaded.records
|
|
12103
|
+
}, options);
|
|
12104
|
+
},
|
|
12105
|
+
setReviewDecision(input) {
|
|
12106
|
+
const decision = {
|
|
12107
|
+
...input,
|
|
12108
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
12109
|
+
};
|
|
12110
|
+
reviewDecisions = [
|
|
12111
|
+
...reviewDecisions.filter((item) => item.item_id !== decision.item_id),
|
|
12112
|
+
decision
|
|
12113
|
+
].sort((left, right) => left.item_id.localeCompare(right.item_id));
|
|
12114
|
+
error = null;
|
|
12115
|
+
notify();
|
|
12116
|
+
return decision;
|
|
12117
|
+
},
|
|
12118
|
+
clearReviewDecision(itemId) {
|
|
12119
|
+
reviewDecisions = reviewDecisions.filter((item) => item.item_id !== itemId);
|
|
12120
|
+
error = null;
|
|
12121
|
+
notify();
|
|
12122
|
+
},
|
|
12123
|
+
createReviewDecisionExport(generatedAt) {
|
|
12124
|
+
const source = index ?? (chunked ? { schema_version: chunked.manifest.source_export_schema_version ?? "aiwg.fortemi.index.export.v1" } : null);
|
|
12125
|
+
if (!source) throw new Error("No AIWG index export or chunked manifest loaded");
|
|
12126
|
+
return createAiwgReviewDecisionExport(source, reviewDecisions, generatedAt);
|
|
12127
|
+
},
|
|
12128
|
+
subscribe(listener) {
|
|
12129
|
+
listeners.add(listener);
|
|
12130
|
+
return () => {
|
|
12131
|
+
listeners.delete(listener);
|
|
12132
|
+
};
|
|
12133
|
+
}
|
|
12134
|
+
};
|
|
12135
|
+
}
|
|
12136
|
+
function aiwgFortemiIndexToCommunityGraph(index, options = {}) {
|
|
12137
|
+
const ids = new Set(index.items.map((item) => item.id));
|
|
12138
|
+
const relationshipWeights = options.relationshipWeights ?? /* @__PURE__ */ Object.create(null);
|
|
12139
|
+
const edgeCounts = /* @__PURE__ */ new Map();
|
|
12140
|
+
for (const item of index.items) {
|
|
12141
|
+
for (const relationship of item.relationships) {
|
|
12142
|
+
if (!ids.has(relationship.target_id) && !options.includeDanglingRelationships) continue;
|
|
12143
|
+
const kind = relationship.type;
|
|
12144
|
+
const configuredWeight = Object.prototype.hasOwnProperty.call(relationshipWeights, kind) ? relationshipWeights[kind] : void 0;
|
|
12145
|
+
const baseWeight = typeof configuredWeight === "number" && Number.isFinite(configuredWeight) ? configuredWeight : 1;
|
|
12146
|
+
const key = `${item.id}\0${relationship.target_id}\0${kind}`;
|
|
12147
|
+
const existing = edgeCounts.get(key);
|
|
12148
|
+
if (existing) existing.weight += baseWeight;
|
|
12149
|
+
else edgeCounts.set(key, { source: item.id, target: relationship.target_id, kind, weight: baseWeight });
|
|
12150
|
+
}
|
|
12151
|
+
}
|
|
12152
|
+
const communities = /* @__PURE__ */ new Map();
|
|
12153
|
+
for (const item of index.items) {
|
|
12154
|
+
const communityIds = communityIdsFor(item, options);
|
|
12155
|
+
for (const communityId of communityIds) {
|
|
12156
|
+
const nodes = communities.get(communityId) ?? [];
|
|
12157
|
+
nodes.push(item.id);
|
|
12158
|
+
communities.set(communityId, nodes);
|
|
12159
|
+
}
|
|
12160
|
+
}
|
|
12161
|
+
return {
|
|
12162
|
+
nodes: index.items.map((item) => ({ id: item.id })),
|
|
12163
|
+
edges: Array.from(edgeCounts.values()).sort((left, right) => left.source.localeCompare(right.source) || left.target.localeCompare(right.target) || left.kind.localeCompare(right.kind)),
|
|
12164
|
+
communities: Array.from(communities.entries()).map(([id, nodes]) => ({ id, nodes: [...new Set(nodes)].sort() })).sort((left, right) => left.id.localeCompare(right.id))
|
|
12165
|
+
};
|
|
12166
|
+
}
|
|
12167
|
+
function communityIdsFor(item, options) {
|
|
12168
|
+
if (options.communityFacet) {
|
|
12169
|
+
const values = item.facets[options.communityFacet] ?? [];
|
|
12170
|
+
if (values.length > 0) return values.map((value) => `${options.communityFacet}:${value}`);
|
|
12171
|
+
}
|
|
12172
|
+
if (options.communityTagPrefix) {
|
|
12173
|
+
const prefix = options.communityTagPrefix;
|
|
12174
|
+
const tags = item.tags.filter((tag) => tag.startsWith(prefix));
|
|
12175
|
+
if (tags.length > 0) return tags;
|
|
12176
|
+
}
|
|
12177
|
+
if (item.concepts.length > 0) return item.concepts.map((concept) => `concept:${concept}`);
|
|
12178
|
+
return [`type:${item.type}`];
|
|
12179
|
+
}
|
|
12180
|
+
var AIWG_SHARD_UUID_NAMESPACE = "7ab5d1f8-29d2-5e35-9e2f-3a45de171a9e";
|
|
12181
|
+
var shardEncoder = new TextEncoder();
|
|
12182
|
+
var shardDecoder = new TextDecoder();
|
|
12183
|
+
function aiwgShardUuid(kind, id) {
|
|
12184
|
+
return v5(`${kind}:${id}`, AIWG_SHARD_UUID_NAMESPACE);
|
|
12185
|
+
}
|
|
12186
|
+
function aiwgShardTimestamp(value, fallback) {
|
|
12187
|
+
if (!value || Number.isNaN(Date.parse(value))) return fallback;
|
|
12188
|
+
return new Date(value).toISOString();
|
|
12189
|
+
}
|
|
12190
|
+
function aiwgRecordTitle(record) {
|
|
12191
|
+
return record.title ?? record.search?.title ?? record.search?.name ?? record.id;
|
|
12192
|
+
}
|
|
12193
|
+
function aiwgRecordContent(record) {
|
|
12194
|
+
if (record.text) return record.text;
|
|
12195
|
+
if (record.search?.body) return record.search.body;
|
|
12196
|
+
const chunks = record.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean);
|
|
12197
|
+
if (chunks?.length) return chunks.join("\n\n");
|
|
12198
|
+
return record.search?.summary ?? "";
|
|
12199
|
+
}
|
|
12200
|
+
function aiwgShardMetadata(index, record) {
|
|
12201
|
+
return {
|
|
12202
|
+
aiwg_fortemi_index: {
|
|
12203
|
+
envelope: {
|
|
12204
|
+
schema_version: index.schema_version,
|
|
12205
|
+
generated_at: index.generated_at,
|
|
12206
|
+
source: index.source,
|
|
12207
|
+
...index.compatibility ? { compatibility: index.compatibility } : {}
|
|
12208
|
+
},
|
|
12209
|
+
record
|
|
12210
|
+
}
|
|
12211
|
+
};
|
|
12212
|
+
}
|
|
12213
|
+
function encodeJsonLines(values) {
|
|
12214
|
+
return shardEncoder.encode(values.map((value) => JSON.stringify(value)).join("\n"));
|
|
12215
|
+
}
|
|
12216
|
+
async function aiwgFortemiIndexToKnowledgeShard(index, options = {}) {
|
|
12217
|
+
const validation = validateAiwgFortemiIndexExport(index);
|
|
12218
|
+
if (!validation.valid) {
|
|
12219
|
+
throw new Error(`Invalid AIWG Fortemi index export:
|
|
12220
|
+
${validation.errors.join("\n")}`);
|
|
12221
|
+
}
|
|
12222
|
+
if (index.schema_version !== "aiwg.fortemi.index.export.v2") {
|
|
12223
|
+
throw new Error("Knowledge Shard conversion requires aiwg.fortemi.index.export.v2");
|
|
12224
|
+
}
|
|
12225
|
+
const createdAt = aiwgShardTimestamp(options.createdAt ?? index.generated_at, (/* @__PURE__ */ new Date()).toISOString());
|
|
12226
|
+
const noteIds = new Map(index.items.map((record) => [record.id, aiwgShardUuid("record", record.id)]));
|
|
12227
|
+
const notes = index.items.map((record) => ({
|
|
12228
|
+
id: noteIds.get(record.id),
|
|
12229
|
+
title: aiwgRecordTitle(record),
|
|
12230
|
+
original_content: aiwgRecordContent(record),
|
|
12231
|
+
revised_content: null,
|
|
12232
|
+
metadata: aiwgShardMetadata(index, record),
|
|
12233
|
+
format: "markdown",
|
|
12234
|
+
source: "aiwg-index",
|
|
12235
|
+
starred: false,
|
|
12236
|
+
archived: false,
|
|
12237
|
+
tags: [...new Set(record.tags)].sort(),
|
|
12238
|
+
created_at: aiwgShardTimestamp(record.source.updated_at ?? record.updated_at, createdAt),
|
|
12239
|
+
updated_at: aiwgShardTimestamp(record.updated_at, createdAt),
|
|
12240
|
+
deleted_at: null
|
|
12241
|
+
}));
|
|
12242
|
+
const links = [];
|
|
12243
|
+
for (const record of index.items) {
|
|
12244
|
+
for (const [position, relationship] of record.relationships.entries()) {
|
|
12245
|
+
const targetNoteId = noteIds.get(relationship.target_id) ?? null;
|
|
12246
|
+
links.push({
|
|
12247
|
+
id: aiwgShardUuid(
|
|
12248
|
+
"relationship",
|
|
12249
|
+
`${record.id}\0${position}\0${relationship.type}\0${relationship.target_id}`
|
|
12250
|
+
),
|
|
12251
|
+
from_note_id: noteIds.get(record.id),
|
|
12252
|
+
to_note_id: targetNoteId,
|
|
12253
|
+
to_url: targetNoteId ? null : `aiwg://record/${encodeURIComponent(relationship.target_id)}`,
|
|
12254
|
+
kind: relationship.type,
|
|
12255
|
+
score: relationship.confidence ?? null,
|
|
12256
|
+
created_at: aiwgShardTimestamp(record.updated_at, createdAt),
|
|
12257
|
+
metadata: {
|
|
12258
|
+
aiwg_fortemi_index: {
|
|
12259
|
+
source_record_id: record.id,
|
|
12260
|
+
target_record_id: relationship.target_id,
|
|
12261
|
+
relationship
|
|
12262
|
+
}
|
|
12263
|
+
}
|
|
12264
|
+
});
|
|
12265
|
+
}
|
|
12266
|
+
}
|
|
12267
|
+
const defaultScheme = index.source.graph ?? index.source.repo;
|
|
12268
|
+
const concepts = /* @__PURE__ */ new Map();
|
|
12269
|
+
for (const record of index.items) {
|
|
12270
|
+
for (const conceptId of record.concepts) {
|
|
12271
|
+
concepts.set(conceptId, {
|
|
12272
|
+
concept: { id: conceptId, prefLabel: conceptId },
|
|
12273
|
+
scheme: defaultScheme
|
|
12274
|
+
});
|
|
12275
|
+
}
|
|
12276
|
+
for (const concept of record.skos_concepts ?? []) {
|
|
12277
|
+
concepts.set(concept.id, { concept, scheme: concept.scheme ?? defaultScheme });
|
|
12278
|
+
}
|
|
12279
|
+
for (const relation of record.skos_relations ?? []) {
|
|
12280
|
+
if (!concepts.has(relation.source_id)) {
|
|
12281
|
+
concepts.set(relation.source_id, {
|
|
12282
|
+
concept: { id: relation.source_id, prefLabel: relation.source_id },
|
|
12283
|
+
scheme: defaultScheme
|
|
12284
|
+
});
|
|
12285
|
+
}
|
|
12286
|
+
if (!concepts.has(relation.target_id)) {
|
|
12287
|
+
concepts.set(relation.target_id, {
|
|
12288
|
+
concept: { id: relation.target_id, prefLabel: relation.target_id },
|
|
12289
|
+
scheme: defaultScheme
|
|
12290
|
+
});
|
|
12291
|
+
}
|
|
12292
|
+
}
|
|
12293
|
+
}
|
|
12294
|
+
const schemeNames = [...new Set([...concepts.values()].map(({ scheme }) => scheme))].sort();
|
|
12295
|
+
const schemes = schemeNames.map((scheme) => ({
|
|
12296
|
+
id: aiwgShardUuid("skos-scheme", scheme),
|
|
12297
|
+
title: scheme,
|
|
12298
|
+
description: `SKOS scheme projected from ${index.source.repo}`,
|
|
12299
|
+
created_at: createdAt,
|
|
12300
|
+
updated_at: createdAt
|
|
12301
|
+
}));
|
|
12302
|
+
const shardConcepts = [...concepts.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([id, { concept, scheme }]) => ({
|
|
12303
|
+
id: aiwgShardUuid("skos-concept", id),
|
|
12304
|
+
scheme_id: aiwgShardUuid("skos-scheme", scheme),
|
|
12305
|
+
pref_label: concept.prefLabel,
|
|
12306
|
+
alt_labels: concept.altLabels ?? [],
|
|
12307
|
+
definition: concept.definition ?? null,
|
|
12308
|
+
created_at: createdAt,
|
|
12309
|
+
updated_at: createdAt
|
|
12310
|
+
}));
|
|
12311
|
+
const noteSkosTags = [];
|
|
12312
|
+
const skosRelations = [];
|
|
12313
|
+
const provenanceEdges = [];
|
|
12314
|
+
for (const record of index.items) {
|
|
12315
|
+
const recordConceptIds = /* @__PURE__ */ new Set([
|
|
12316
|
+
...record.concepts,
|
|
12317
|
+
...(record.skos_concepts ?? []).map((concept) => concept.id)
|
|
12318
|
+
]);
|
|
12319
|
+
for (const conceptId of [...recordConceptIds].sort()) {
|
|
12320
|
+
noteSkosTags.push({
|
|
12321
|
+
id: aiwgShardUuid("note-skos-tag", `${record.id}\0${conceptId}`),
|
|
12322
|
+
note_id: noteIds.get(record.id),
|
|
12323
|
+
concept_id: aiwgShardUuid("skos-concept", conceptId),
|
|
12324
|
+
created_at: createdAt
|
|
12325
|
+
});
|
|
12326
|
+
}
|
|
12327
|
+
for (const [position, relation] of (record.skos_relations ?? []).entries()) {
|
|
12328
|
+
const relationType = relation.type === "broader" || relation.type === "narrower" ? relation.type : "related";
|
|
12329
|
+
skosRelations.push({
|
|
12330
|
+
id: aiwgShardUuid(
|
|
12331
|
+
"skos-relation",
|
|
12332
|
+
`${record.id}\0${position}\0${relation.source_id}\0${relation.type}\0${relation.target_id}`
|
|
12333
|
+
),
|
|
12334
|
+
source_concept_id: aiwgShardUuid("skos-concept", relation.source_id),
|
|
12335
|
+
target_concept_id: aiwgShardUuid("skos-concept", relation.target_id),
|
|
12336
|
+
relation_type: relationType,
|
|
12337
|
+
created_at: createdAt
|
|
12338
|
+
});
|
|
12339
|
+
}
|
|
12340
|
+
for (const [position, provenance] of record.provenance.entries()) {
|
|
12341
|
+
provenanceEdges.push({
|
|
12342
|
+
id: aiwgShardUuid("provenance", `${record.id}\0field\0${position}`),
|
|
12343
|
+
entity_type: "note",
|
|
12344
|
+
entity_id: noteIds.get(record.id),
|
|
12345
|
+
activity: "source",
|
|
12346
|
+
agent: provenance.source,
|
|
12347
|
+
started_at: aiwgShardTimestamp(record.updated_at, createdAt),
|
|
12348
|
+
ended_at: null,
|
|
12349
|
+
attributes: { aiwg_fortemi_index: { provenance } }
|
|
12350
|
+
});
|
|
12351
|
+
}
|
|
12352
|
+
for (const [position, event] of (record.provenance_events ?? []).entries()) {
|
|
12353
|
+
provenanceEdges.push({
|
|
12354
|
+
id: aiwgShardUuid("provenance", event.id ?? `${record.id}\0event\0${position}`),
|
|
12355
|
+
entity_type: "note",
|
|
12356
|
+
entity_id: noteIds.get(record.id),
|
|
12357
|
+
activity: event.activity,
|
|
12358
|
+
agent: event.agent ?? "aiwg-index",
|
|
12359
|
+
started_at: aiwgShardTimestamp(event.started_at, createdAt),
|
|
12360
|
+
ended_at: event.ended_at ? aiwgShardTimestamp(event.ended_at, createdAt) : null,
|
|
12361
|
+
attributes: { aiwg_fortemi_index: { event } }
|
|
12362
|
+
});
|
|
12363
|
+
}
|
|
12364
|
+
}
|
|
12365
|
+
const files = /* @__PURE__ */ new Map();
|
|
12366
|
+
const components = ["notes", "tags"];
|
|
12367
|
+
const counts = {
|
|
12368
|
+
notes: notes.length,
|
|
12369
|
+
tags: [...new Set(notes.flatMap((note) => note.tags))].length
|
|
12370
|
+
};
|
|
12371
|
+
files.set("notes.jsonl", encodeJsonLines(notes));
|
|
12372
|
+
files.set("tags.json", shardEncoder.encode(JSON.stringify(
|
|
12373
|
+
[...new Set(notes.flatMap((note) => note.tags))].sort().map((name) => ({ name, created_at: createdAt }))
|
|
12374
|
+
)));
|
|
12375
|
+
const addComponent = (component, filename, values, jsonLines = true) => {
|
|
12376
|
+
if (values.length === 0) return;
|
|
12377
|
+
components.push(component);
|
|
12378
|
+
counts[component] = values.length;
|
|
12379
|
+
files.set(filename, jsonLines ? encodeJsonLines(values) : shardEncoder.encode(JSON.stringify(values)));
|
|
12380
|
+
};
|
|
12381
|
+
addComponent("links", "links.jsonl", links);
|
|
12382
|
+
if (options.includeNativeRichComponents) {
|
|
12383
|
+
addComponent("skos_schemes", "skos_schemes.json", schemes, false);
|
|
12384
|
+
addComponent("skos_concepts", "skos_concepts.json", shardConcepts, false);
|
|
12385
|
+
addComponent("skos_relations", "skos_relations.jsonl", skosRelations);
|
|
12386
|
+
addComponent("note_skos_tags", "note_skos_tags.jsonl", noteSkosTags);
|
|
12387
|
+
addComponent("provenance_edges", "provenance_edges.jsonl", provenanceEdges);
|
|
12388
|
+
}
|
|
12389
|
+
const checksums = {};
|
|
12390
|
+
for (const [filename, bytes] of files) checksums[filename] = await sha256Hex(bytes);
|
|
12391
|
+
const manifest = {
|
|
12392
|
+
version: CURRENT_SHARD_VERSION,
|
|
12393
|
+
matric_version: options.matricVersion ?? "fortemi-core-aiwg-index",
|
|
12394
|
+
format: SHARD_FORMAT,
|
|
12395
|
+
created_at: createdAt,
|
|
12396
|
+
components,
|
|
12397
|
+
counts,
|
|
12398
|
+
checksums,
|
|
12399
|
+
min_reader_version: CURRENT_SHARD_VERSION,
|
|
12400
|
+
migrated_from: null,
|
|
12401
|
+
migration_history: []
|
|
12402
|
+
};
|
|
12403
|
+
files.set("manifest.json", shardEncoder.encode(JSON.stringify(manifest, null, 2)));
|
|
12404
|
+
return packTarGz(files);
|
|
12405
|
+
}
|
|
12406
|
+
function aiwgFortemiIndexFromKnowledgeShard(bytes) {
|
|
12407
|
+
const files = unpackTarGz(bytes);
|
|
12408
|
+
const noteBytes = files.get("notes.jsonl");
|
|
12409
|
+
const notes = (noteBytes ? shardDecoder.decode(noteBytes) : "").split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
12410
|
+
if (notes.length === 0) throw new Error("Knowledge Shard contains no AIWG index notes");
|
|
12411
|
+
let envelope;
|
|
12412
|
+
const items = [];
|
|
12413
|
+
for (const note of notes) {
|
|
12414
|
+
const metadata = note.metadata?.aiwg_fortemi_index;
|
|
12415
|
+
if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
|
|
12416
|
+
throw new Error(`Knowledge Shard note ${note.id} has no AIWG index metadata`);
|
|
12417
|
+
}
|
|
12418
|
+
const value = metadata;
|
|
12419
|
+
envelope ??= value.envelope;
|
|
12420
|
+
items.push(value.record);
|
|
12421
|
+
}
|
|
12422
|
+
const restored = {
|
|
12423
|
+
...envelope,
|
|
12424
|
+
items: items.sort((left, right) => left.id.localeCompare(right.id))
|
|
12425
|
+
};
|
|
12426
|
+
const validation = validateAiwgFortemiIndexExport(restored);
|
|
12427
|
+
if (!validation.valid) {
|
|
12428
|
+
throw new Error(`Knowledge Shard contains invalid AIWG index metadata:
|
|
12429
|
+
${validation.errors.join("\n")}`);
|
|
12430
|
+
}
|
|
12431
|
+
return restored;
|
|
12432
|
+
}
|
|
12433
|
+
|
|
12434
|
+
// schemas/aiwg-fortemi-index-export.schema.json
|
|
12435
|
+
var aiwg_fortemi_index_export_schema_default = {
|
|
12436
|
+
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
12437
|
+
$id: "https://aiwg.io/schemas/aiwg-fortemi-index-export.json",
|
|
12438
|
+
title: "AIWG Fortemi Index Export",
|
|
12439
|
+
type: "object",
|
|
12440
|
+
additionalProperties: false,
|
|
12441
|
+
required: ["schema_version", "generated_at", "source", "items"],
|
|
12442
|
+
properties: {
|
|
12443
|
+
schema_version: {
|
|
12444
|
+
enum: ["aiwg.fortemi.index.export.v1", "aiwg.fortemi.index.export.v2"]
|
|
12445
|
+
},
|
|
12446
|
+
generated_at: {
|
|
12447
|
+
type: "string",
|
|
12448
|
+
format: "date-time"
|
|
12449
|
+
},
|
|
12450
|
+
source: {
|
|
12451
|
+
type: "object",
|
|
12452
|
+
additionalProperties: false,
|
|
12453
|
+
required: ["repo", "privacy"],
|
|
12454
|
+
properties: {
|
|
12455
|
+
repo: {
|
|
12456
|
+
type: "string",
|
|
12457
|
+
minLength: 1
|
|
12458
|
+
},
|
|
12459
|
+
privacy: {
|
|
12460
|
+
$ref: "#/$defs/privacy"
|
|
12461
|
+
},
|
|
12462
|
+
graph: {
|
|
12463
|
+
type: "string",
|
|
12464
|
+
minLength: 1
|
|
12465
|
+
}
|
|
12466
|
+
}
|
|
12467
|
+
},
|
|
12468
|
+
compatibility: {
|
|
12469
|
+
type: "object",
|
|
12470
|
+
additionalProperties: false,
|
|
12471
|
+
required: ["previous_schema_version", "strategy"],
|
|
12472
|
+
properties: {
|
|
12473
|
+
previous_schema_version: {
|
|
12474
|
+
const: "aiwg.fortemi.index.export.v1"
|
|
12475
|
+
},
|
|
12476
|
+
strategy: {
|
|
12477
|
+
const: "supported"
|
|
12478
|
+
}
|
|
12479
|
+
}
|
|
12480
|
+
},
|
|
12481
|
+
items: {
|
|
12482
|
+
type: "array",
|
|
12483
|
+
items: {
|
|
12484
|
+
$ref: "#/$defs/record"
|
|
12485
|
+
}
|
|
12486
|
+
}
|
|
12487
|
+
},
|
|
12488
|
+
allOf: [
|
|
12489
|
+
{
|
|
12490
|
+
if: {
|
|
12491
|
+
properties: {
|
|
12492
|
+
schema_version: {
|
|
12493
|
+
const: "aiwg.fortemi.index.export.v1"
|
|
12494
|
+
}
|
|
12495
|
+
}
|
|
12496
|
+
},
|
|
12497
|
+
then: {
|
|
12498
|
+
not: {
|
|
12499
|
+
required: ["compatibility"]
|
|
12500
|
+
},
|
|
12501
|
+
properties: {
|
|
12502
|
+
source: {
|
|
12503
|
+
not: {
|
|
12504
|
+
required: ["graph"]
|
|
12505
|
+
}
|
|
12506
|
+
},
|
|
12507
|
+
items: {
|
|
12508
|
+
items: {
|
|
12509
|
+
allOf: [
|
|
12510
|
+
{
|
|
12511
|
+
properties: {
|
|
12512
|
+
schema_version: {
|
|
12513
|
+
const: "aiwg.fortemi.index.record.v1"
|
|
12514
|
+
}
|
|
12515
|
+
}
|
|
12516
|
+
},
|
|
12517
|
+
{
|
|
12518
|
+
$ref: "#/$defs/v1RecordCompatibility"
|
|
12519
|
+
}
|
|
12520
|
+
]
|
|
12521
|
+
}
|
|
12522
|
+
}
|
|
12523
|
+
}
|
|
12524
|
+
}
|
|
12525
|
+
},
|
|
12526
|
+
{
|
|
12527
|
+
if: {
|
|
12528
|
+
properties: {
|
|
12529
|
+
schema_version: {
|
|
12530
|
+
const: "aiwg.fortemi.index.export.v2"
|
|
12531
|
+
}
|
|
12532
|
+
}
|
|
12533
|
+
},
|
|
12534
|
+
then: {
|
|
12535
|
+
required: ["compatibility"],
|
|
12536
|
+
properties: {
|
|
12537
|
+
source: {
|
|
12538
|
+
required: ["graph"]
|
|
12539
|
+
},
|
|
12540
|
+
items: {
|
|
12541
|
+
items: {
|
|
12542
|
+
properties: {
|
|
12543
|
+
schema_version: {
|
|
12544
|
+
const: "aiwg.fortemi.index.record.v2"
|
|
12545
|
+
}
|
|
12546
|
+
}
|
|
12547
|
+
}
|
|
12548
|
+
}
|
|
12549
|
+
}
|
|
12550
|
+
}
|
|
12551
|
+
}
|
|
12552
|
+
],
|
|
12553
|
+
$defs: {
|
|
12554
|
+
privacy: {
|
|
12555
|
+
enum: ["private", "sanitized", "public"]
|
|
12556
|
+
},
|
|
12557
|
+
recordType: {
|
|
12558
|
+
type: "string",
|
|
12559
|
+
minLength: 1
|
|
12560
|
+
},
|
|
12561
|
+
stringArray: {
|
|
12562
|
+
type: "array",
|
|
12563
|
+
items: {
|
|
12564
|
+
type: "string"
|
|
12565
|
+
}
|
|
12566
|
+
},
|
|
12567
|
+
numberArray: {
|
|
12568
|
+
type: "array",
|
|
12569
|
+
items: {
|
|
12570
|
+
type: "number"
|
|
12571
|
+
}
|
|
12572
|
+
},
|
|
12573
|
+
v1RecordCompatibility: {
|
|
12574
|
+
type: "object",
|
|
12575
|
+
not: {
|
|
12576
|
+
anyOf: [
|
|
12577
|
+
{
|
|
12578
|
+
required: ["name"]
|
|
12579
|
+
},
|
|
12580
|
+
{
|
|
12581
|
+
required: ["summary"]
|
|
12582
|
+
},
|
|
12583
|
+
{
|
|
12584
|
+
required: ["search"]
|
|
12585
|
+
},
|
|
12586
|
+
{
|
|
12587
|
+
required: ["chunks"]
|
|
12588
|
+
},
|
|
12589
|
+
{
|
|
12590
|
+
required: ["embeddings"]
|
|
12591
|
+
},
|
|
12592
|
+
{
|
|
12593
|
+
required: ["compatibility"]
|
|
12594
|
+
},
|
|
12595
|
+
{
|
|
12596
|
+
required: ["skos_concepts"]
|
|
12597
|
+
},
|
|
12598
|
+
{
|
|
12599
|
+
required: ["skos_relations"]
|
|
12600
|
+
},
|
|
12601
|
+
{
|
|
12602
|
+
required: ["provenance_events"]
|
|
12603
|
+
}
|
|
12604
|
+
]
|
|
12605
|
+
},
|
|
12606
|
+
properties: {
|
|
12607
|
+
source: {
|
|
12608
|
+
not: {
|
|
12609
|
+
anyOf: [
|
|
12610
|
+
{
|
|
12611
|
+
required: ["origin"]
|
|
12612
|
+
},
|
|
12613
|
+
{
|
|
12614
|
+
required: ["generated"]
|
|
12615
|
+
},
|
|
12616
|
+
{
|
|
12617
|
+
required: ["checksum"]
|
|
12618
|
+
},
|
|
12619
|
+
{
|
|
12620
|
+
required: ["updated_at"]
|
|
12621
|
+
}
|
|
12622
|
+
]
|
|
12623
|
+
}
|
|
12624
|
+
},
|
|
12625
|
+
relationships: {
|
|
12626
|
+
items: {
|
|
12627
|
+
not: {
|
|
12628
|
+
anyOf: [
|
|
12629
|
+
{
|
|
12630
|
+
required: ["target_path"]
|
|
12631
|
+
},
|
|
12632
|
+
{
|
|
12633
|
+
required: ["direction"]
|
|
12634
|
+
},
|
|
12635
|
+
{
|
|
12636
|
+
required: ["label"]
|
|
12637
|
+
},
|
|
12638
|
+
{
|
|
12639
|
+
required: ["confidence"]
|
|
12640
|
+
},
|
|
12641
|
+
{
|
|
12642
|
+
required: ["privacy"]
|
|
12643
|
+
},
|
|
12644
|
+
{
|
|
12645
|
+
required: ["metadata"]
|
|
12646
|
+
}
|
|
12647
|
+
]
|
|
12648
|
+
}
|
|
12649
|
+
}
|
|
12650
|
+
},
|
|
12651
|
+
privacy: {
|
|
12652
|
+
not: {
|
|
12653
|
+
required: ["locality"]
|
|
12654
|
+
}
|
|
12655
|
+
}
|
|
12656
|
+
}
|
|
12657
|
+
},
|
|
12658
|
+
record: {
|
|
12659
|
+
type: "object",
|
|
12660
|
+
additionalProperties: false,
|
|
12661
|
+
required: [
|
|
12662
|
+
"schema_version",
|
|
12663
|
+
"id",
|
|
12664
|
+
"type",
|
|
12665
|
+
"source",
|
|
12666
|
+
"title",
|
|
12667
|
+
"text",
|
|
12668
|
+
"facets",
|
|
12669
|
+
"tags",
|
|
12670
|
+
"concepts",
|
|
12671
|
+
"relationships",
|
|
12672
|
+
"provenance",
|
|
12673
|
+
"privacy",
|
|
12674
|
+
"updated_at"
|
|
12675
|
+
],
|
|
12676
|
+
properties: {
|
|
12677
|
+
schema_version: {
|
|
12678
|
+
enum: [
|
|
12679
|
+
"aiwg.fortemi.index.record.v1",
|
|
12680
|
+
"aiwg.fortemi.index.record.v2"
|
|
12681
|
+
]
|
|
12682
|
+
},
|
|
12683
|
+
id: {
|
|
12684
|
+
type: "string",
|
|
12685
|
+
minLength: 1
|
|
12686
|
+
},
|
|
12687
|
+
type: {
|
|
12688
|
+
$ref: "#/$defs/recordType"
|
|
12689
|
+
},
|
|
12690
|
+
source: {
|
|
12691
|
+
type: "object",
|
|
12692
|
+
additionalProperties: false,
|
|
12693
|
+
required: ["path", "repo_relative_path", "locator"],
|
|
12694
|
+
properties: {
|
|
12695
|
+
path: {
|
|
12696
|
+
type: "string",
|
|
12697
|
+
minLength: 1
|
|
12698
|
+
},
|
|
12699
|
+
repo_relative_path: {
|
|
12700
|
+
type: "string",
|
|
12701
|
+
minLength: 1
|
|
12702
|
+
},
|
|
12703
|
+
locator: {
|
|
12704
|
+
type: "string",
|
|
12705
|
+
minLength: 1
|
|
12706
|
+
},
|
|
12707
|
+
origin: {
|
|
12708
|
+
type: "string",
|
|
12709
|
+
minLength: 1
|
|
12710
|
+
},
|
|
12711
|
+
generated: {
|
|
12712
|
+
type: "boolean"
|
|
12713
|
+
},
|
|
12714
|
+
checksum: {
|
|
12715
|
+
type: "string"
|
|
12716
|
+
},
|
|
12717
|
+
updated_at: {
|
|
12718
|
+
type: "string",
|
|
12719
|
+
format: "date-time"
|
|
12720
|
+
}
|
|
12721
|
+
}
|
|
12722
|
+
},
|
|
12723
|
+
title: {
|
|
12724
|
+
type: "string"
|
|
12725
|
+
},
|
|
12726
|
+
name: {
|
|
12727
|
+
type: "string"
|
|
12728
|
+
},
|
|
12729
|
+
summary: {
|
|
12730
|
+
type: "string"
|
|
12731
|
+
},
|
|
12732
|
+
text: {
|
|
12733
|
+
type: "string"
|
|
12734
|
+
},
|
|
12735
|
+
search: {
|
|
12736
|
+
type: "object",
|
|
12737
|
+
additionalProperties: false,
|
|
12738
|
+
required: [
|
|
12739
|
+
"title",
|
|
12740
|
+
"body",
|
|
12741
|
+
"triggers",
|
|
12742
|
+
"aliases",
|
|
12743
|
+
"tags",
|
|
12744
|
+
"frontmatter"
|
|
12745
|
+
],
|
|
12746
|
+
properties: {
|
|
12747
|
+
title: {
|
|
12748
|
+
type: "string"
|
|
12749
|
+
},
|
|
12750
|
+
name: {
|
|
12751
|
+
type: "string"
|
|
12752
|
+
},
|
|
12753
|
+
summary: {
|
|
12754
|
+
type: "string"
|
|
12755
|
+
},
|
|
12756
|
+
body: {
|
|
12757
|
+
type: "string"
|
|
12758
|
+
},
|
|
12759
|
+
triggers: {
|
|
12760
|
+
$ref: "#/$defs/stringArray"
|
|
12761
|
+
},
|
|
12762
|
+
aliases: {
|
|
12763
|
+
$ref: "#/$defs/stringArray"
|
|
12764
|
+
},
|
|
12765
|
+
capability: {
|
|
12766
|
+
type: "string"
|
|
12767
|
+
},
|
|
12768
|
+
tags: {
|
|
12769
|
+
$ref: "#/$defs/stringArray"
|
|
12770
|
+
},
|
|
12771
|
+
phase: {
|
|
12772
|
+
type: "string"
|
|
12773
|
+
},
|
|
12774
|
+
type: {
|
|
12775
|
+
type: "string"
|
|
12776
|
+
},
|
|
12777
|
+
frontmatter: {
|
|
12778
|
+
type: "object"
|
|
12779
|
+
}
|
|
12780
|
+
}
|
|
12781
|
+
},
|
|
12782
|
+
facets: {
|
|
12783
|
+
type: "object",
|
|
12784
|
+
additionalProperties: {
|
|
12785
|
+
$ref: "#/$defs/stringArray"
|
|
12786
|
+
}
|
|
12787
|
+
},
|
|
12788
|
+
tags: {
|
|
12789
|
+
$ref: "#/$defs/stringArray"
|
|
12790
|
+
},
|
|
12791
|
+
concepts: {
|
|
12792
|
+
$ref: "#/$defs/stringArray"
|
|
12793
|
+
},
|
|
12794
|
+
relationships: {
|
|
12795
|
+
type: "array",
|
|
12796
|
+
items: {
|
|
12797
|
+
type: "object",
|
|
12798
|
+
additionalProperties: false,
|
|
12799
|
+
required: ["type", "target_id"],
|
|
12800
|
+
properties: {
|
|
12801
|
+
type: {
|
|
12802
|
+
type: "string",
|
|
12803
|
+
minLength: 1
|
|
12804
|
+
},
|
|
12805
|
+
target_id: {
|
|
12806
|
+
type: "string",
|
|
12807
|
+
minLength: 1
|
|
12808
|
+
},
|
|
12809
|
+
source_path: {
|
|
12810
|
+
type: "string"
|
|
12811
|
+
},
|
|
12812
|
+
target_path: {
|
|
12813
|
+
type: "string"
|
|
12814
|
+
},
|
|
12815
|
+
direction: {
|
|
12816
|
+
enum: ["upstream", "downstream", "related"]
|
|
12817
|
+
},
|
|
12818
|
+
label: {
|
|
12819
|
+
type: "string"
|
|
12820
|
+
},
|
|
12821
|
+
confidence: {
|
|
12822
|
+
type: "number"
|
|
12823
|
+
},
|
|
12824
|
+
privacy: {
|
|
12825
|
+
$ref: "#/$defs/privacy"
|
|
12826
|
+
},
|
|
12827
|
+
metadata: {
|
|
12828
|
+
type: "object"
|
|
12829
|
+
}
|
|
12830
|
+
}
|
|
12831
|
+
}
|
|
12832
|
+
},
|
|
12833
|
+
provenance: {
|
|
12834
|
+
type: "array",
|
|
12835
|
+
minItems: 1,
|
|
12836
|
+
items: {
|
|
12837
|
+
type: "object",
|
|
12838
|
+
additionalProperties: false,
|
|
12839
|
+
required: ["field", "source", "path", "confidence", "privacy"],
|
|
12840
|
+
properties: {
|
|
12841
|
+
field: {
|
|
12842
|
+
type: "string",
|
|
12843
|
+
minLength: 1
|
|
12844
|
+
},
|
|
12845
|
+
source: {
|
|
12846
|
+
type: "string",
|
|
12847
|
+
minLength: 1
|
|
12848
|
+
},
|
|
12849
|
+
path: {
|
|
12850
|
+
type: "string",
|
|
12851
|
+
minLength: 1
|
|
12852
|
+
},
|
|
12853
|
+
confidence: {
|
|
12854
|
+
enum: ["source", "candidate", "reviewed", "rejected"]
|
|
12855
|
+
},
|
|
12856
|
+
privacy: {
|
|
12857
|
+
$ref: "#/$defs/privacy"
|
|
12858
|
+
}
|
|
12859
|
+
}
|
|
12860
|
+
}
|
|
12861
|
+
},
|
|
12862
|
+
privacy: {
|
|
12863
|
+
type: "object",
|
|
12864
|
+
additionalProperties: false,
|
|
12865
|
+
required: ["classification", "pii"],
|
|
12866
|
+
properties: {
|
|
12867
|
+
classification: {
|
|
12868
|
+
$ref: "#/$defs/privacy"
|
|
12869
|
+
},
|
|
12870
|
+
pii: {
|
|
12871
|
+
type: "boolean"
|
|
12872
|
+
},
|
|
12873
|
+
locality: {
|
|
12874
|
+
enum: ["project", "framework", "external"]
|
|
12875
|
+
}
|
|
12876
|
+
}
|
|
12877
|
+
},
|
|
12878
|
+
chunks: {
|
|
12879
|
+
type: "array",
|
|
12880
|
+
items: {
|
|
12881
|
+
type: "object",
|
|
12882
|
+
additionalProperties: false,
|
|
12883
|
+
properties: {
|
|
12884
|
+
id: {
|
|
12885
|
+
type: "string",
|
|
12886
|
+
minLength: 1
|
|
12887
|
+
},
|
|
12888
|
+
text: {
|
|
12889
|
+
type: "string"
|
|
12890
|
+
},
|
|
12891
|
+
body: {
|
|
12892
|
+
type: "string"
|
|
12893
|
+
},
|
|
12894
|
+
summary: {
|
|
12895
|
+
type: "string"
|
|
12896
|
+
},
|
|
12897
|
+
source_path: {
|
|
12898
|
+
type: "string"
|
|
12899
|
+
},
|
|
12900
|
+
metadata: {
|
|
12901
|
+
type: "object"
|
|
12902
|
+
},
|
|
12903
|
+
checksum: {
|
|
12904
|
+
type: "string"
|
|
12905
|
+
}
|
|
12906
|
+
}
|
|
12907
|
+
}
|
|
12908
|
+
},
|
|
12909
|
+
embeddings: {
|
|
12910
|
+
type: "array",
|
|
12911
|
+
items: {
|
|
12912
|
+
type: "object",
|
|
12913
|
+
additionalProperties: false,
|
|
12914
|
+
properties: {
|
|
12915
|
+
id: {
|
|
12916
|
+
type: "string"
|
|
12917
|
+
},
|
|
12918
|
+
model: {
|
|
12919
|
+
type: "string"
|
|
12920
|
+
},
|
|
12921
|
+
embedding: {
|
|
12922
|
+
$ref: "#/$defs/numberArray"
|
|
12923
|
+
},
|
|
12924
|
+
vector: {
|
|
12925
|
+
$ref: "#/$defs/numberArray"
|
|
12926
|
+
},
|
|
12927
|
+
granularity: {
|
|
12928
|
+
type: "string"
|
|
12929
|
+
},
|
|
12930
|
+
source_path: {
|
|
12931
|
+
type: "string"
|
|
12932
|
+
},
|
|
12933
|
+
metadata: {
|
|
12934
|
+
type: "object"
|
|
12935
|
+
},
|
|
12936
|
+
chunk_id: {
|
|
12937
|
+
type: "string"
|
|
12938
|
+
},
|
|
12939
|
+
vector_ref: {
|
|
12940
|
+
type: "string"
|
|
12941
|
+
},
|
|
12942
|
+
input_hash: {
|
|
12943
|
+
type: "string"
|
|
12944
|
+
}
|
|
12945
|
+
}
|
|
12946
|
+
}
|
|
12947
|
+
},
|
|
12948
|
+
compatibility: {
|
|
12949
|
+
type: "object"
|
|
12950
|
+
},
|
|
12951
|
+
skos_concepts: {
|
|
12952
|
+
type: "array",
|
|
12953
|
+
items: {
|
|
12954
|
+
type: "object",
|
|
12955
|
+
additionalProperties: false,
|
|
12956
|
+
required: ["id", "prefLabel"],
|
|
12957
|
+
properties: {
|
|
12958
|
+
id: {
|
|
12959
|
+
type: "string",
|
|
12960
|
+
minLength: 1
|
|
12961
|
+
},
|
|
12962
|
+
prefLabel: {
|
|
12963
|
+
type: "string",
|
|
12964
|
+
minLength: 1
|
|
12965
|
+
},
|
|
12966
|
+
definition: {
|
|
12967
|
+
type: "string"
|
|
12968
|
+
},
|
|
12969
|
+
scheme: {
|
|
12970
|
+
type: "string"
|
|
12971
|
+
},
|
|
12972
|
+
notation: {
|
|
12973
|
+
type: "string"
|
|
12974
|
+
},
|
|
12975
|
+
uri: {
|
|
12976
|
+
type: "string"
|
|
12977
|
+
},
|
|
12978
|
+
altLabels: {
|
|
12979
|
+
$ref: "#/$defs/stringArray"
|
|
12980
|
+
},
|
|
12981
|
+
metadata: {
|
|
12982
|
+
type: "object"
|
|
12983
|
+
}
|
|
12984
|
+
}
|
|
12985
|
+
}
|
|
12986
|
+
},
|
|
12987
|
+
skos_relations: {
|
|
12988
|
+
type: "array",
|
|
12989
|
+
items: {
|
|
12990
|
+
type: "object",
|
|
12991
|
+
additionalProperties: false,
|
|
12992
|
+
required: ["type", "source_id", "target_id"],
|
|
12993
|
+
properties: {
|
|
12994
|
+
type: {
|
|
12995
|
+
type: "string",
|
|
12996
|
+
minLength: 1
|
|
12997
|
+
},
|
|
12998
|
+
source_id: {
|
|
12999
|
+
type: "string",
|
|
13000
|
+
minLength: 1
|
|
13001
|
+
},
|
|
13002
|
+
target_id: {
|
|
13003
|
+
type: "string",
|
|
13004
|
+
minLength: 1
|
|
13005
|
+
},
|
|
13006
|
+
source_path: {
|
|
13007
|
+
type: "string"
|
|
13008
|
+
},
|
|
13009
|
+
metadata: {
|
|
13010
|
+
type: "object"
|
|
13011
|
+
}
|
|
13012
|
+
}
|
|
13013
|
+
}
|
|
13014
|
+
},
|
|
13015
|
+
provenance_events: {
|
|
13016
|
+
type: "array",
|
|
13017
|
+
items: {
|
|
13018
|
+
type: "object",
|
|
13019
|
+
additionalProperties: false,
|
|
13020
|
+
required: ["activity"],
|
|
13021
|
+
properties: {
|
|
13022
|
+
id: {
|
|
13023
|
+
type: "string"
|
|
13024
|
+
},
|
|
13025
|
+
activity: {
|
|
13026
|
+
type: "string",
|
|
13027
|
+
minLength: 1
|
|
13028
|
+
},
|
|
13029
|
+
agent: {
|
|
13030
|
+
type: "string"
|
|
13031
|
+
},
|
|
13032
|
+
started_at: {
|
|
13033
|
+
type: "string",
|
|
13034
|
+
format: "date-time"
|
|
13035
|
+
},
|
|
13036
|
+
ended_at: {
|
|
13037
|
+
type: "string",
|
|
13038
|
+
format: "date-time"
|
|
13039
|
+
},
|
|
13040
|
+
source: {
|
|
13041
|
+
type: "string"
|
|
13042
|
+
},
|
|
13043
|
+
path: {
|
|
13044
|
+
type: "string"
|
|
13045
|
+
},
|
|
13046
|
+
confidence: {
|
|
13047
|
+
enum: ["source", "candidate", "reviewed", "rejected"]
|
|
13048
|
+
},
|
|
13049
|
+
privacy: {
|
|
13050
|
+
$ref: "#/$defs/privacy"
|
|
13051
|
+
},
|
|
13052
|
+
attributes: {
|
|
13053
|
+
type: "object"
|
|
13054
|
+
}
|
|
13055
|
+
}
|
|
13056
|
+
}
|
|
13057
|
+
},
|
|
13058
|
+
updated_at: {
|
|
13059
|
+
type: "string",
|
|
13060
|
+
format: "date-time"
|
|
13061
|
+
}
|
|
13062
|
+
}
|
|
13063
|
+
}
|
|
13064
|
+
}
|
|
13065
|
+
};
|
|
13066
|
+
|
|
13067
|
+
// src/aiwg-index-schema.ts
|
|
13068
|
+
var PROJECTED_FIELDS = [
|
|
13069
|
+
"schema_version",
|
|
13070
|
+
"id",
|
|
13071
|
+
"type",
|
|
13072
|
+
"title",
|
|
13073
|
+
"text",
|
|
13074
|
+
"facets",
|
|
13075
|
+
"tags",
|
|
13076
|
+
"concepts",
|
|
13077
|
+
"privacy"
|
|
13078
|
+
];
|
|
13079
|
+
var ajvInstance2;
|
|
13080
|
+
var exportValidator;
|
|
13081
|
+
var projectedRecordValidator;
|
|
13082
|
+
function getAiwgFortemiIndexExportSchema() {
|
|
13083
|
+
return aiwg_fortemi_index_export_schema_default;
|
|
11903
13084
|
}
|
|
11904
|
-
|
|
11905
|
-
|
|
11906
|
-
|
|
11907
|
-
|
|
11908
|
-
|
|
11909
|
-
|
|
11910
|
-
|
|
11911
|
-
|
|
11912
|
-
const items = [];
|
|
11913
|
-
for (const partRef of parts) {
|
|
11914
|
-
const loaded = await loadChunkPart(runtime, partRef);
|
|
11915
|
-
if (loaded.fetched) fetchedParts += 1;
|
|
11916
|
-
scannedParts += 1;
|
|
11917
|
-
options.onProgress?.({ phase: "part", done: scannedParts, total: parts.length, href: partRef.href });
|
|
11918
|
-
const start = Math.max(0, offset - partRef.offset);
|
|
11919
|
-
const end = Math.min(loaded.part.items.length, offset + limit - partRef.offset);
|
|
11920
|
-
items.push(...loaded.part.items.slice(start, end));
|
|
11921
|
-
}
|
|
11922
|
-
return {
|
|
11923
|
-
items,
|
|
11924
|
-
total: runtime.manifest.total,
|
|
11925
|
-
facets: runtime.manifest.facets ?? {},
|
|
11926
|
-
manifestTotal: runtime.manifest.total,
|
|
11927
|
-
scannedParts,
|
|
11928
|
-
fetchedParts,
|
|
11929
|
-
complete: true
|
|
11930
|
-
};
|
|
11931
|
-
}
|
|
11932
|
-
const matchKey = matchSetCacheKey(q, options);
|
|
11933
|
-
const cached = runtime.matchCache.get(matchKey);
|
|
11934
|
-
if (cached) {
|
|
11935
|
-
runtime.matchCache.delete(matchKey);
|
|
11936
|
-
runtime.matchCache.set(matchKey, cached);
|
|
11937
|
-
return {
|
|
11938
|
-
...createQueryResultFromRankedEntries(cached, q, options),
|
|
11939
|
-
manifestTotal: runtime.manifest.total,
|
|
11940
|
-
scannedParts: 0,
|
|
11941
|
-
fetchedParts: 0,
|
|
11942
|
-
complete: true
|
|
11943
|
-
};
|
|
11944
|
-
}
|
|
11945
|
-
const entries = [];
|
|
11946
|
-
for (const partRef of runtime.manifest.parts) {
|
|
11947
|
-
const loaded = await loadChunkPart(runtime, partRef);
|
|
11948
|
-
if (loaded.fetched) fetchedParts += 1;
|
|
11949
|
-
scannedParts += 1;
|
|
11950
|
-
options.onProgress?.({ phase: "part", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
|
|
11951
|
-
entries.push(...createRankedEntries(loaded.part.items, q, options, partRef.offset));
|
|
11952
|
-
options.onProgress?.({ phase: "query", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
|
|
13085
|
+
function getAjv2() {
|
|
13086
|
+
if (!ajvInstance2) {
|
|
13087
|
+
ajvInstance2 = new Ajv2020({
|
|
13088
|
+
allErrors: true,
|
|
13089
|
+
strict: false,
|
|
13090
|
+
validateFormats: false
|
|
13091
|
+
});
|
|
13092
|
+
ajvInstance2.addSchema(aiwg_fortemi_index_export_schema_default);
|
|
11953
13093
|
}
|
|
11954
|
-
|
|
11955
|
-
return {
|
|
11956
|
-
...createQueryResultFromRankedEntries(entries, q, options),
|
|
11957
|
-
manifestTotal: runtime.manifest.total,
|
|
11958
|
-
scannedParts,
|
|
11959
|
-
fetchedParts,
|
|
11960
|
-
complete: true
|
|
11961
|
-
};
|
|
11962
|
-
}
|
|
11963
|
-
function createAiwgReviewDecisionExport(source, decisions, generatedAt = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
11964
|
-
return {
|
|
11965
|
-
schema_version: "aiwg.fortemi.review-decisions.v1",
|
|
11966
|
-
generated_at: generatedAt,
|
|
11967
|
-
source_export_schema_version: source.schema_version,
|
|
11968
|
-
decisions: [...decisions].sort((left, right) => left.item_id.localeCompare(right.item_id))
|
|
11969
|
-
};
|
|
13094
|
+
return ajvInstance2;
|
|
11970
13095
|
}
|
|
11971
|
-
function
|
|
11972
|
-
|
|
11973
|
-
|
|
11974
|
-
|
|
11975
|
-
let error = null;
|
|
11976
|
-
let reviewDecisions = [];
|
|
11977
|
-
const listeners = /* @__PURE__ */ new Set();
|
|
11978
|
-
const snapshot = () => ({
|
|
11979
|
-
index,
|
|
11980
|
-
chunked: chunked ? {
|
|
11981
|
-
manifest: chunked.manifest,
|
|
11982
|
-
cachedParts: chunked.partCache.size,
|
|
11983
|
-
maxCachedParts: chunked.maxCachedParts
|
|
11984
|
-
} : null,
|
|
11985
|
-
data,
|
|
11986
|
-
error,
|
|
11987
|
-
reviewDecisions: [...reviewDecisions]
|
|
13096
|
+
function formatErrors2(errors) {
|
|
13097
|
+
return (errors ?? []).map((error) => {
|
|
13098
|
+
const path = error.instancePath || "(root)";
|
|
13099
|
+
return `${path} ${error.message ?? "is invalid"}`;
|
|
11988
13100
|
});
|
|
11989
|
-
|
|
11990
|
-
|
|
11991
|
-
|
|
11992
|
-
|
|
11993
|
-
|
|
11994
|
-
|
|
11995
|
-
|
|
11996
|
-
|
|
11997
|
-
|
|
11998
|
-
|
|
11999
|
-
|
|
12000
|
-
|
|
12001
|
-
|
|
12002
|
-
|
|
12003
|
-
|
|
12004
|
-
|
|
12005
|
-
|
|
12006
|
-
|
|
12007
|
-
|
|
12008
|
-
|
|
12009
|
-
|
|
12010
|
-
|
|
12011
|
-
|
|
12012
|
-
|
|
12013
|
-
|
|
12014
|
-
|
|
12015
|
-
|
|
12016
|
-
|
|
12017
|
-
|
|
12018
|
-
|
|
12019
|
-
|
|
12020
|
-
|
|
12021
|
-
|
|
12022
|
-
|
|
12023
|
-
|
|
12024
|
-
|
|
12025
|
-
|
|
12026
|
-
|
|
12027
|
-
|
|
12028
|
-
|
|
12029
|
-
|
|
12030
|
-
|
|
12031
|
-
|
|
12032
|
-
|
|
12033
|
-
|
|
12034
|
-
|
|
12035
|
-
|
|
12036
|
-
|
|
12037
|
-
|
|
12038
|
-
|
|
12039
|
-
|
|
12040
|
-
|
|
12041
|
-
|
|
12042
|
-
|
|
12043
|
-
|
|
12044
|
-
|
|
12045
|
-
|
|
12046
|
-
|
|
12047
|
-
|
|
12048
|
-
|
|
12049
|
-
|
|
12050
|
-
|
|
12051
|
-
|
|
12052
|
-
|
|
12053
|
-
|
|
12054
|
-
|
|
12055
|
-
|
|
12056
|
-
|
|
12057
|
-
|
|
12058
|
-
|
|
12059
|
-
|
|
12060
|
-
|
|
12061
|
-
|
|
12062
|
-
|
|
12063
|
-
|
|
12064
|
-
|
|
12065
|
-
|
|
12066
|
-
|
|
12067
|
-
|
|
12068
|
-
|
|
12069
|
-
|
|
12070
|
-
|
|
12071
|
-
|
|
12072
|
-
|
|
12073
|
-
|
|
12074
|
-
|
|
12075
|
-
|
|
12076
|
-
|
|
12077
|
-
|
|
13101
|
+
}
|
|
13102
|
+
function getExportValidator() {
|
|
13103
|
+
exportValidator ??= getAjv2().getSchema(aiwg_fortemi_index_export_schema_default.$id) ?? getAjv2().compile(aiwg_fortemi_index_export_schema_default);
|
|
13104
|
+
return exportValidator;
|
|
13105
|
+
}
|
|
13106
|
+
function getProjectedRecordValidator() {
|
|
13107
|
+
if (!projectedRecordValidator) {
|
|
13108
|
+
const properties = Object.fromEntries(Object.keys(aiwg_fortemi_index_export_schema_default.$defs.record.properties).map((field) => [
|
|
13109
|
+
field,
|
|
13110
|
+
{ $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/record/properties/${field}` }
|
|
13111
|
+
]));
|
|
13112
|
+
projectedRecordValidator = getAjv2().compile({
|
|
13113
|
+
type: "object",
|
|
13114
|
+
required: PROJECTED_FIELDS,
|
|
13115
|
+
properties,
|
|
13116
|
+
additionalProperties: true,
|
|
13117
|
+
allOf: [
|
|
13118
|
+
{
|
|
13119
|
+
if: {
|
|
13120
|
+
properties: {
|
|
13121
|
+
schema_version: { const: "aiwg.fortemi.index.record.v1" }
|
|
13122
|
+
}
|
|
13123
|
+
},
|
|
13124
|
+
then: { $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/v1RecordCompatibility` }
|
|
13125
|
+
}
|
|
13126
|
+
]
|
|
13127
|
+
});
|
|
13128
|
+
}
|
|
13129
|
+
return projectedRecordValidator;
|
|
13130
|
+
}
|
|
13131
|
+
function validateAiwgFortemiIndexExportSchema(value) {
|
|
13132
|
+
const validate = getExportValidator();
|
|
13133
|
+
const schemaValue = value && typeof value === "object" && Array.isArray(value.items) ? {
|
|
13134
|
+
...value,
|
|
13135
|
+
items: value.items.map((item) => {
|
|
13136
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) return item;
|
|
13137
|
+
const schemaRecord = { ...item };
|
|
13138
|
+
Reflect.deleteProperty(schemaRecord, "binary_sources");
|
|
13139
|
+
return schemaRecord;
|
|
13140
|
+
})
|
|
13141
|
+
} : value;
|
|
13142
|
+
const valid = validate(schemaValue);
|
|
13143
|
+
return { valid, errors: formatErrors2(validate.errors) };
|
|
13144
|
+
}
|
|
13145
|
+
function validateAiwgFortemiProjectedRecordSchema(value) {
|
|
13146
|
+
const validate = getProjectedRecordValidator();
|
|
13147
|
+
const valid = validate(value);
|
|
13148
|
+
return { valid, errors: formatErrors2(validate.errors) };
|
|
13149
|
+
}
|
|
13150
|
+
|
|
13151
|
+
// src/records/types.ts
|
|
13152
|
+
var RECORD_COLLECTIONS = [
|
|
13153
|
+
"note",
|
|
13154
|
+
"note_original",
|
|
13155
|
+
"note_revised_current",
|
|
13156
|
+
"note_tag",
|
|
13157
|
+
"link",
|
|
13158
|
+
"collection",
|
|
13159
|
+
"collection_note",
|
|
13160
|
+
"attachment",
|
|
13161
|
+
"attachment_blob"
|
|
13162
|
+
];
|
|
13163
|
+
var RECORD_STORE_CAPABILITIES = {
|
|
13164
|
+
crud: true,
|
|
13165
|
+
journal: true,
|
|
13166
|
+
boundedTextScan: true,
|
|
13167
|
+
fullTextSearch: false,
|
|
13168
|
+
vectorSearch: false,
|
|
13169
|
+
sqlJoins: false
|
|
13170
|
+
};
|
|
13171
|
+
|
|
13172
|
+
// src/records/memory-record-store.ts
|
|
13173
|
+
var MemoryRecordStore = class {
|
|
13174
|
+
capabilities = RECORD_STORE_CAPABILITIES;
|
|
13175
|
+
collections = /* @__PURE__ */ new Map();
|
|
13176
|
+
journal = [];
|
|
13177
|
+
seq = 0;
|
|
13178
|
+
table(collection) {
|
|
13179
|
+
let table = this.collections.get(collection);
|
|
13180
|
+
if (!table) {
|
|
13181
|
+
table = /* @__PURE__ */ new Map();
|
|
13182
|
+
this.collections.set(collection, table);
|
|
13183
|
+
}
|
|
13184
|
+
return table;
|
|
13185
|
+
}
|
|
13186
|
+
async get(collection, id) {
|
|
13187
|
+
return this.table(collection).get(id) ?? null;
|
|
13188
|
+
}
|
|
13189
|
+
async put(collection, record) {
|
|
13190
|
+
const entry = {
|
|
13191
|
+
seq: ++this.seq,
|
|
13192
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13193
|
+
op: "put",
|
|
13194
|
+
collection,
|
|
13195
|
+
id: record.id,
|
|
13196
|
+
record: structuredClone(record)
|
|
13197
|
+
};
|
|
13198
|
+
this.table(collection).set(record.id, structuredClone(record));
|
|
13199
|
+
this.journal.push(entry);
|
|
13200
|
+
return entry;
|
|
13201
|
+
}
|
|
13202
|
+
async remove(collection, id) {
|
|
13203
|
+
const entry = {
|
|
13204
|
+
seq: ++this.seq,
|
|
13205
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13206
|
+
op: "delete",
|
|
13207
|
+
collection,
|
|
13208
|
+
id
|
|
13209
|
+
};
|
|
13210
|
+
this.table(collection).delete(id);
|
|
13211
|
+
this.journal.push(entry);
|
|
13212
|
+
return entry;
|
|
13213
|
+
}
|
|
13214
|
+
async list(collection, opts) {
|
|
13215
|
+
const all = [...this.table(collection).values()];
|
|
13216
|
+
return opts?.limit !== void 0 ? all.slice(0, opts.limit) : all;
|
|
13217
|
+
}
|
|
13218
|
+
async journalSince(sinceSeq, limit) {
|
|
13219
|
+
const entries = this.journal.filter((e) => e.seq > sinceSeq);
|
|
13220
|
+
return limit !== void 0 ? entries.slice(0, limit) : entries;
|
|
13221
|
+
}
|
|
13222
|
+
async headSeq() {
|
|
13223
|
+
return this.seq;
|
|
13224
|
+
}
|
|
13225
|
+
async close() {
|
|
13226
|
+
}
|
|
13227
|
+
};
|
|
13228
|
+
|
|
13229
|
+
// src/records/idb-record-store.ts
|
|
13230
|
+
var DB_VERSION = 1;
|
|
13231
|
+
var JOURNAL_STORE = "journal";
|
|
13232
|
+
var META_STORE = "meta";
|
|
13233
|
+
var RECORD_SCHEMA_VERSION = 1;
|
|
13234
|
+
function requestToPromise(request) {
|
|
13235
|
+
return new Promise((resolve, reject) => {
|
|
13236
|
+
request.onsuccess = () => resolve(request.result);
|
|
13237
|
+
request.onerror = () => reject(request.error);
|
|
13238
|
+
});
|
|
13239
|
+
}
|
|
13240
|
+
function transactionComplete(tx) {
|
|
13241
|
+
return new Promise((resolve, reject) => {
|
|
13242
|
+
tx.oncomplete = () => resolve();
|
|
13243
|
+
tx.onabort = () => reject(tx.error);
|
|
13244
|
+
tx.onerror = () => reject(tx.error);
|
|
13245
|
+
});
|
|
13246
|
+
}
|
|
13247
|
+
var IdbRecordStore = class _IdbRecordStore {
|
|
13248
|
+
constructor(db) {
|
|
13249
|
+
this.db = db;
|
|
13250
|
+
}
|
|
13251
|
+
capabilities = RECORD_STORE_CAPABILITIES;
|
|
13252
|
+
static async open(archiveName, options) {
|
|
13253
|
+
const factory = options?.indexedDB ?? globalThis.indexedDB;
|
|
13254
|
+
if (!factory) {
|
|
13255
|
+
throw new Error("IdbRecordStore requires IndexedDB (none available in this environment)");
|
|
13256
|
+
}
|
|
13257
|
+
const db = await new Promise((resolve, reject) => {
|
|
13258
|
+
const request = factory.open(`fortemi-${archiveName}-records`, DB_VERSION);
|
|
13259
|
+
request.onupgradeneeded = () => {
|
|
13260
|
+
const database = request.result;
|
|
13261
|
+
for (const collection of RECORD_COLLECTIONS) {
|
|
13262
|
+
if (!database.objectStoreNames.contains(collection)) {
|
|
13263
|
+
database.createObjectStore(collection, { keyPath: "id" });
|
|
13264
|
+
}
|
|
13265
|
+
}
|
|
13266
|
+
if (!database.objectStoreNames.contains(JOURNAL_STORE)) {
|
|
13267
|
+
database.createObjectStore(JOURNAL_STORE, { keyPath: "seq", autoIncrement: true });
|
|
13268
|
+
}
|
|
13269
|
+
if (!database.objectStoreNames.contains(META_STORE)) {
|
|
13270
|
+
database.createObjectStore(META_STORE);
|
|
12078
13271
|
}
|
|
12079
|
-
}
|
|
12080
|
-
const found = requireIndex().items.find((item) => item.id === id);
|
|
12081
|
-
if (!found) throw new Error("Record not found: " + id);
|
|
12082
|
-
return found;
|
|
12083
|
-
},
|
|
12084
|
-
async neighbors(id, options) {
|
|
12085
|
-
try {
|
|
12086
|
-
const queryOptions = neighborQueryOptions(id, options);
|
|
12087
|
-
const result = chunked ? await relationshipResultFromChunkedRuntime(chunked, queryOptions) : relationshipResultFromRecords(requireIndex().items, queryOptions);
|
|
12088
|
-
return filterNeighborResult(id, result, options);
|
|
12089
|
-
} catch (err) {
|
|
12090
|
-
error = err instanceof Error ? err : new Error(String(err));
|
|
12091
|
-
notify();
|
|
12092
|
-
throw error;
|
|
12093
|
-
}
|
|
12094
|
-
},
|
|
12095
|
-
async relationshipQuery(options) {
|
|
12096
|
-
try {
|
|
12097
|
-
return chunked ? await relationshipResultFromChunkedRuntime(chunked, options) : relationshipResultFromRecords(requireIndex().items, options);
|
|
12098
|
-
} catch (err) {
|
|
12099
|
-
error = err instanceof Error ? err : new Error(String(err));
|
|
12100
|
-
notify();
|
|
12101
|
-
throw error;
|
|
12102
|
-
}
|
|
12103
|
-
},
|
|
12104
|
-
async relationshipSet(options) {
|
|
12105
|
-
const [left, right] = await Promise.all([
|
|
12106
|
-
this.neighbors(options.a, options),
|
|
12107
|
-
this.neighbors(options.b, options)
|
|
12108
|
-
]);
|
|
12109
|
-
const leftIds = new Set(left.nodes.map((node) => node.id).filter((id) => id !== options.a));
|
|
12110
|
-
const rightIds = new Set(right.nodes.map((node) => node.id).filter((id) => id !== options.b));
|
|
12111
|
-
let ids;
|
|
12112
|
-
if (options.op === "intersection") ids = [...leftIds].filter((id) => rightIds.has(id));
|
|
12113
|
-
else if (options.op === "difference") ids = [...leftIds].filter((id) => !rightIds.has(id));
|
|
12114
|
-
else ids = [.../* @__PURE__ */ new Set([...leftIds, ...rightIds])];
|
|
12115
|
-
return { op: options.op, ids: ids.sort() };
|
|
12116
|
-
},
|
|
12117
|
-
clearChunkCache() {
|
|
12118
|
-
chunked?.partCache.clear();
|
|
12119
|
-
chunked?.detailCache.clear();
|
|
12120
|
-
chunked?.matchCache.clear();
|
|
12121
|
-
error = null;
|
|
12122
|
-
notify();
|
|
12123
|
-
},
|
|
12124
|
-
toCommunityGraph(options) {
|
|
12125
|
-
return aiwgFortemiIndexToCommunityGraph(requireIndex(), options);
|
|
12126
|
-
},
|
|
12127
|
-
async toCommunityGraphChunked(options) {
|
|
12128
|
-
if (!chunked) return aiwgFortemiIndexToCommunityGraph(requireIndex(), options);
|
|
12129
|
-
const loaded = await recordsFromChunkedRuntime(chunked, options?.onProgress);
|
|
12130
|
-
return aiwgFortemiIndexToCommunityGraph({
|
|
12131
|
-
generated_at: chunked.manifest.generated_at,
|
|
12132
|
-
source: chunked.manifest.source,
|
|
12133
|
-
items: loaded.records
|
|
12134
|
-
}, options);
|
|
12135
|
-
},
|
|
12136
|
-
setReviewDecision(input) {
|
|
12137
|
-
const decision = {
|
|
12138
|
-
...input,
|
|
12139
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
12140
|
-
};
|
|
12141
|
-
reviewDecisions = [
|
|
12142
|
-
...reviewDecisions.filter((item) => item.item_id !== decision.item_id),
|
|
12143
|
-
decision
|
|
12144
|
-
].sort((left, right) => left.item_id.localeCompare(right.item_id));
|
|
12145
|
-
error = null;
|
|
12146
|
-
notify();
|
|
12147
|
-
return decision;
|
|
12148
|
-
},
|
|
12149
|
-
clearReviewDecision(itemId) {
|
|
12150
|
-
reviewDecisions = reviewDecisions.filter((item) => item.item_id !== itemId);
|
|
12151
|
-
error = null;
|
|
12152
|
-
notify();
|
|
12153
|
-
},
|
|
12154
|
-
createReviewDecisionExport(generatedAt) {
|
|
12155
|
-
const source = index ?? (chunked ? { schema_version: chunked.manifest.source_export_schema_version ?? "aiwg.fortemi.index.export.v1" } : null);
|
|
12156
|
-
if (!source) throw new Error("No AIWG index export or chunked manifest loaded");
|
|
12157
|
-
return createAiwgReviewDecisionExport(source, reviewDecisions, generatedAt);
|
|
12158
|
-
},
|
|
12159
|
-
subscribe(listener) {
|
|
12160
|
-
listeners.add(listener);
|
|
12161
|
-
return () => {
|
|
12162
|
-
listeners.delete(listener);
|
|
12163
13272
|
};
|
|
13273
|
+
request.onsuccess = () => resolve(request.result);
|
|
13274
|
+
request.onerror = () => reject(request.error);
|
|
13275
|
+
});
|
|
13276
|
+
const store = new _IdbRecordStore(db);
|
|
13277
|
+
await store.ensureSchemaVersion();
|
|
13278
|
+
return store;
|
|
13279
|
+
}
|
|
13280
|
+
async ensureSchemaVersion() {
|
|
13281
|
+
const tx = this.db.transaction(META_STORE, "readwrite");
|
|
13282
|
+
const meta = tx.objectStore(META_STORE);
|
|
13283
|
+
const current = await requestToPromise(meta.get("schemaVersion"));
|
|
13284
|
+
if (current === void 0) {
|
|
13285
|
+
meta.put(RECORD_SCHEMA_VERSION, "schemaVersion");
|
|
13286
|
+
} else if (current > RECORD_SCHEMA_VERSION) {
|
|
13287
|
+
throw new Error(
|
|
13288
|
+
`IdbRecordStore: records were written by a newer schema (v${current} > v${RECORD_SCHEMA_VERSION}); refusing to open`
|
|
13289
|
+
);
|
|
12164
13290
|
}
|
|
12165
|
-
|
|
13291
|
+
await transactionComplete(tx);
|
|
13292
|
+
}
|
|
13293
|
+
async get(collection, id) {
|
|
13294
|
+
const tx = this.db.transaction(collection, "readonly");
|
|
13295
|
+
const result = await requestToPromise(
|
|
13296
|
+
tx.objectStore(collection).get(id)
|
|
13297
|
+
);
|
|
13298
|
+
return result ?? null;
|
|
13299
|
+
}
|
|
13300
|
+
async put(collection, record) {
|
|
13301
|
+
const tx = this.db.transaction([collection, JOURNAL_STORE], "readwrite");
|
|
13302
|
+
tx.objectStore(collection).put(record);
|
|
13303
|
+
const pending = {
|
|
13304
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13305
|
+
op: "put",
|
|
13306
|
+
collection,
|
|
13307
|
+
id: record.id,
|
|
13308
|
+
record
|
|
13309
|
+
};
|
|
13310
|
+
const seqReq = tx.objectStore(JOURNAL_STORE).add(pending);
|
|
13311
|
+
await transactionComplete(tx);
|
|
13312
|
+
return { ...pending, seq: seqReq.result };
|
|
13313
|
+
}
|
|
13314
|
+
async remove(collection, id) {
|
|
13315
|
+
const tx = this.db.transaction([collection, JOURNAL_STORE], "readwrite");
|
|
13316
|
+
tx.objectStore(collection).delete(id);
|
|
13317
|
+
const pending = {
|
|
13318
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13319
|
+
op: "delete",
|
|
13320
|
+
collection,
|
|
13321
|
+
id
|
|
13322
|
+
};
|
|
13323
|
+
const seqReq = tx.objectStore(JOURNAL_STORE).add(pending);
|
|
13324
|
+
await transactionComplete(tx);
|
|
13325
|
+
return { ...pending, seq: seqReq.result };
|
|
13326
|
+
}
|
|
13327
|
+
async list(collection, opts) {
|
|
13328
|
+
const tx = this.db.transaction(collection, "readonly");
|
|
13329
|
+
const store = tx.objectStore(collection);
|
|
13330
|
+
const all = await requestToPromise(
|
|
13331
|
+
opts?.limit !== void 0 ? store.getAll(void 0, opts.limit) : store.getAll()
|
|
13332
|
+
);
|
|
13333
|
+
return all;
|
|
13334
|
+
}
|
|
13335
|
+
async journalSince(sinceSeq, limit) {
|
|
13336
|
+
const tx = this.db.transaction(JOURNAL_STORE, "readonly");
|
|
13337
|
+
const range = IDBKeyRange.lowerBound(sinceSeq, true);
|
|
13338
|
+
const entries = await requestToPromise(
|
|
13339
|
+
limit !== void 0 ? tx.objectStore(JOURNAL_STORE).getAll(range, limit) : tx.objectStore(JOURNAL_STORE).getAll(range)
|
|
13340
|
+
);
|
|
13341
|
+
return entries;
|
|
13342
|
+
}
|
|
13343
|
+
async headSeq() {
|
|
13344
|
+
const tx = this.db.transaction(JOURNAL_STORE, "readonly");
|
|
13345
|
+
const cursor = await requestToPromise(
|
|
13346
|
+
tx.objectStore(JOURNAL_STORE).openCursor(null, "prev")
|
|
13347
|
+
);
|
|
13348
|
+
return cursor ? cursor.value.seq : 0;
|
|
13349
|
+
}
|
|
13350
|
+
async close() {
|
|
13351
|
+
this.db.close();
|
|
13352
|
+
}
|
|
13353
|
+
};
|
|
13354
|
+
function createRecordStore(archiveName, options) {
|
|
13355
|
+
return IdbRecordStore.open(archiveName, options);
|
|
12166
13356
|
}
|
|
12167
|
-
|
|
12168
|
-
|
|
12169
|
-
|
|
12170
|
-
|
|
12171
|
-
|
|
12172
|
-
|
|
12173
|
-
|
|
12174
|
-
|
|
12175
|
-
|
|
12176
|
-
|
|
12177
|
-
|
|
12178
|
-
|
|
12179
|
-
|
|
12180
|
-
|
|
13357
|
+
|
|
13358
|
+
// src/records/canonical-notes-repository.ts
|
|
13359
|
+
function nowIso() {
|
|
13360
|
+
return (/* @__PURE__ */ new Date()).toISOString();
|
|
13361
|
+
}
|
|
13362
|
+
var CanonicalNotesRepository = class {
|
|
13363
|
+
constructor(store) {
|
|
13364
|
+
this.store = store;
|
|
13365
|
+
}
|
|
13366
|
+
// ── Notes ─────────────────────────────────────────────────────────────────
|
|
13367
|
+
async create(input) {
|
|
13368
|
+
const noteId = input.id ?? generateId();
|
|
13369
|
+
const ts = nowIso();
|
|
13370
|
+
await this.store.put("note", {
|
|
13371
|
+
id: noteId,
|
|
13372
|
+
archive_id: null,
|
|
13373
|
+
title: input.title ?? null,
|
|
13374
|
+
format: input.format ?? "markdown",
|
|
13375
|
+
source: input.source ?? "user",
|
|
13376
|
+
visibility: input.visibility ?? "private",
|
|
13377
|
+
revision_mode: "standard",
|
|
13378
|
+
is_starred: false,
|
|
13379
|
+
is_pinned: false,
|
|
13380
|
+
is_archived: false,
|
|
13381
|
+
created_at: ts,
|
|
13382
|
+
updated_at: ts,
|
|
13383
|
+
deleted_at: null
|
|
13384
|
+
});
|
|
13385
|
+
await this.store.put("note_original", {
|
|
13386
|
+
id: generateId(),
|
|
13387
|
+
note_id: noteId,
|
|
13388
|
+
content: input.content,
|
|
13389
|
+
content_hash: computeHash(new TextEncoder().encode(input.content)),
|
|
13390
|
+
created_at: ts
|
|
13391
|
+
});
|
|
13392
|
+
await this.store.put("note_revised_current", {
|
|
13393
|
+
id: noteId,
|
|
13394
|
+
content: input.content,
|
|
13395
|
+
ai_metadata: null,
|
|
13396
|
+
generation_count: 0,
|
|
13397
|
+
model: null,
|
|
13398
|
+
is_user_edited: false,
|
|
13399
|
+
updated_at: ts
|
|
13400
|
+
});
|
|
13401
|
+
return await this.get(noteId);
|
|
13402
|
+
}
|
|
13403
|
+
async get(noteId) {
|
|
13404
|
+
const note = await this.store.get("note", noteId);
|
|
13405
|
+
if (!note) return null;
|
|
13406
|
+
const revised = await this.store.get("note_revised_current", noteId);
|
|
13407
|
+
const originals = (await this.store.list("note_original")).filter(
|
|
13408
|
+
(o) => o.note_id === noteId
|
|
13409
|
+
);
|
|
13410
|
+
const tags = (await this.store.list("note_tag")).filter((t) => t.note_id === noteId).map((t) => t.tag).sort();
|
|
13411
|
+
return {
|
|
13412
|
+
note,
|
|
13413
|
+
original_content: originals[0]?.content ?? "",
|
|
13414
|
+
revised_content: revised?.content ?? originals[0]?.content ?? "",
|
|
13415
|
+
tags
|
|
13416
|
+
};
|
|
13417
|
+
}
|
|
13418
|
+
async update(noteId, input) {
|
|
13419
|
+
const note = await this.store.get("note", noteId);
|
|
13420
|
+
if (!note) throw new Error(`Note not found: ${noteId}`);
|
|
13421
|
+
const ts = nowIso();
|
|
13422
|
+
await this.store.put("note", {
|
|
13423
|
+
...note,
|
|
13424
|
+
title: input.title !== void 0 ? input.title : note.title,
|
|
13425
|
+
is_starred: input.is_starred ?? note.is_starred,
|
|
13426
|
+
is_pinned: input.is_pinned ?? note.is_pinned,
|
|
13427
|
+
is_archived: input.is_archived ?? note.is_archived,
|
|
13428
|
+
updated_at: ts
|
|
13429
|
+
});
|
|
13430
|
+
if (input.content !== void 0) {
|
|
13431
|
+
const revised = await this.store.get("note_revised_current", noteId);
|
|
13432
|
+
await this.store.put("note_revised_current", {
|
|
13433
|
+
id: noteId,
|
|
13434
|
+
content: input.content,
|
|
13435
|
+
ai_metadata: revised?.ai_metadata ?? null,
|
|
13436
|
+
generation_count: revised?.generation_count ?? 0,
|
|
13437
|
+
model: revised?.model ?? null,
|
|
13438
|
+
is_user_edited: true,
|
|
13439
|
+
updated_at: ts
|
|
13440
|
+
});
|
|
12181
13441
|
}
|
|
13442
|
+
return await this.get(noteId);
|
|
12182
13443
|
}
|
|
12183
|
-
|
|
12184
|
-
|
|
12185
|
-
const
|
|
12186
|
-
|
|
12187
|
-
|
|
12188
|
-
|
|
12189
|
-
|
|
13444
|
+
/** Soft-delete: sets `deleted_at`; the record (and history) remains. */
|
|
13445
|
+
async softDelete(noteId) {
|
|
13446
|
+
const note = await this.store.get("note", noteId);
|
|
13447
|
+
if (!note) throw new Error(`Note not found: ${noteId}`);
|
|
13448
|
+
await this.store.put("note", { ...note, deleted_at: nowIso(), updated_at: nowIso() });
|
|
13449
|
+
}
|
|
13450
|
+
async restore(noteId) {
|
|
13451
|
+
const note = await this.store.get("note", noteId);
|
|
13452
|
+
if (!note) throw new Error(`Note not found: ${noteId}`);
|
|
13453
|
+
await this.store.put("note", { ...note, deleted_at: null, updated_at: nowIso() });
|
|
13454
|
+
}
|
|
13455
|
+
/** Non-deleted notes, most recently updated first. */
|
|
13456
|
+
async listRecent(limit = 50) {
|
|
13457
|
+
const notes = (await this.store.list("note")).filter((n) => n.deleted_at === null);
|
|
13458
|
+
notes.sort((a, b) => b.updated_at.localeCompare(a.updated_at));
|
|
13459
|
+
return notes.slice(0, limit);
|
|
13460
|
+
}
|
|
13461
|
+
/**
|
|
13462
|
+
* Bounded substring scan over title + revised content (case-insensitive).
|
|
13463
|
+
* This is deliberately not ranked FTS — see `store.capabilities`.
|
|
13464
|
+
*/
|
|
13465
|
+
async searchText(query, limit = 20) {
|
|
13466
|
+
const needle = query.toLowerCase();
|
|
13467
|
+
const revised = new Map(
|
|
13468
|
+
(await this.store.list("note_revised_current")).map((r) => [r.id, r.content])
|
|
13469
|
+
);
|
|
13470
|
+
const hits = [];
|
|
13471
|
+
for (const note of await this.store.list("note")) {
|
|
13472
|
+
if (note.deleted_at !== null) continue;
|
|
13473
|
+
const haystack = `${note.title ?? ""}
|
|
13474
|
+
${revised.get(note.id) ?? ""}`.toLowerCase();
|
|
13475
|
+
if (haystack.includes(needle)) {
|
|
13476
|
+
hits.push(note);
|
|
13477
|
+
if (hits.length >= limit) break;
|
|
13478
|
+
}
|
|
12190
13479
|
}
|
|
13480
|
+
return hits;
|
|
12191
13481
|
}
|
|
12192
|
-
|
|
12193
|
-
|
|
12194
|
-
|
|
12195
|
-
|
|
12196
|
-
|
|
13482
|
+
// ── Tags ──────────────────────────────────────────────────────────────────
|
|
13483
|
+
async addTag(noteId, tag) {
|
|
13484
|
+
const existing = (await this.store.list("note_tag")).find(
|
|
13485
|
+
(t) => t.note_id === noteId && t.tag === tag
|
|
13486
|
+
);
|
|
13487
|
+
if (existing) return;
|
|
13488
|
+
await this.store.put("note_tag", {
|
|
13489
|
+
id: generateId(),
|
|
13490
|
+
note_id: noteId,
|
|
13491
|
+
tag,
|
|
13492
|
+
created_at: nowIso()
|
|
13493
|
+
});
|
|
13494
|
+
}
|
|
13495
|
+
async removeTag(noteId, tag) {
|
|
13496
|
+
const existing = (await this.store.list("note_tag")).find(
|
|
13497
|
+
(t) => t.note_id === noteId && t.tag === tag
|
|
13498
|
+
);
|
|
13499
|
+
if (existing) await this.store.remove("note_tag", existing.id);
|
|
13500
|
+
}
|
|
13501
|
+
async notesByTag(tag) {
|
|
13502
|
+
const noteIds = new Set(
|
|
13503
|
+
(await this.store.list("note_tag")).filter((t) => t.tag === tag).map((t) => t.note_id)
|
|
13504
|
+
);
|
|
13505
|
+
return (await this.store.list("note")).filter(
|
|
13506
|
+
(n) => noteIds.has(n.id) && n.deleted_at === null
|
|
13507
|
+
);
|
|
13508
|
+
}
|
|
13509
|
+
// ── Links ─────────────────────────────────────────────────────────────────
|
|
13510
|
+
async createLink(sourceNoteId, targetNoteId, linkType = "related") {
|
|
13511
|
+
const link = {
|
|
13512
|
+
id: generateId(),
|
|
13513
|
+
source_note_id: sourceNoteId,
|
|
13514
|
+
target_note_id: targetNoteId,
|
|
13515
|
+
link_type: linkType,
|
|
13516
|
+
created_at: nowIso(),
|
|
13517
|
+
deleted_at: null
|
|
13518
|
+
};
|
|
13519
|
+
await this.store.put("link", link);
|
|
13520
|
+
return link;
|
|
13521
|
+
}
|
|
13522
|
+
async softDeleteLink(linkId) {
|
|
13523
|
+
const link = await this.store.get("link", linkId);
|
|
13524
|
+
if (!link) throw new Error(`Link not found: ${linkId}`);
|
|
13525
|
+
await this.store.put("link", { ...link, deleted_at: nowIso() });
|
|
13526
|
+
}
|
|
13527
|
+
/** Active links touching a note (either direction). */
|
|
13528
|
+
async linksOf(noteId) {
|
|
13529
|
+
return (await this.store.list("link")).filter(
|
|
13530
|
+
(l) => l.deleted_at === null && (l.source_note_id === noteId || l.target_note_id === noteId)
|
|
13531
|
+
);
|
|
13532
|
+
}
|
|
13533
|
+
// ── Collections ───────────────────────────────────────────────────────────
|
|
13534
|
+
async createCollection(name, description) {
|
|
13535
|
+
const ts = nowIso();
|
|
13536
|
+
const collection = {
|
|
13537
|
+
id: generateId(),
|
|
13538
|
+
name,
|
|
13539
|
+
description: description ?? null,
|
|
13540
|
+
created_at: ts,
|
|
13541
|
+
updated_at: ts,
|
|
13542
|
+
deleted_at: null
|
|
13543
|
+
};
|
|
13544
|
+
await this.store.put("collection", collection);
|
|
13545
|
+
return collection;
|
|
13546
|
+
}
|
|
13547
|
+
async addNoteToCollection(collectionId, noteId) {
|
|
13548
|
+
const existing = (await this.store.list("collection_note")).find(
|
|
13549
|
+
(cn) => cn.collection_id === collectionId && cn.note_id === noteId
|
|
13550
|
+
);
|
|
13551
|
+
if (existing) return;
|
|
13552
|
+
await this.store.put("collection_note", {
|
|
13553
|
+
id: generateId(),
|
|
13554
|
+
collection_id: collectionId,
|
|
13555
|
+
note_id: noteId,
|
|
13556
|
+
created_at: nowIso()
|
|
13557
|
+
});
|
|
13558
|
+
}
|
|
13559
|
+
async notesInCollection(collectionId) {
|
|
13560
|
+
const noteIds = new Set(
|
|
13561
|
+
(await this.store.list("collection_note")).filter((cn) => cn.collection_id === collectionId).map((cn) => cn.note_id)
|
|
13562
|
+
);
|
|
13563
|
+
return (await this.store.list("note")).filter(
|
|
13564
|
+
(n) => noteIds.has(n.id) && n.deleted_at === null
|
|
13565
|
+
);
|
|
13566
|
+
}
|
|
13567
|
+
};
|
|
13568
|
+
|
|
13569
|
+
// src/records/canonical-attachments-repository.ts
|
|
13570
|
+
function nowIso2() {
|
|
13571
|
+
return (/* @__PURE__ */ new Date()).toISOString();
|
|
12197
13572
|
}
|
|
12198
|
-
|
|
12199
|
-
|
|
12200
|
-
|
|
12201
|
-
|
|
13573
|
+
var CanonicalAttachmentsRepository = class {
|
|
13574
|
+
constructor(store, blobStore) {
|
|
13575
|
+
this.store = store;
|
|
13576
|
+
this.blobStore = blobStore;
|
|
12202
13577
|
}
|
|
12203
|
-
|
|
12204
|
-
|
|
12205
|
-
const
|
|
12206
|
-
|
|
13578
|
+
/** Bytes-first attach (ADR-013 D5); dedupes on the store-computed hash. */
|
|
13579
|
+
async attach(input) {
|
|
13580
|
+
const contentHash = await this.blobStore.put(input.data);
|
|
13581
|
+
let blob = (await this.store.list("attachment_blob")).find(
|
|
13582
|
+
(b) => b.content_hash === contentHash
|
|
13583
|
+
);
|
|
13584
|
+
if (!blob) {
|
|
13585
|
+
blob = {
|
|
13586
|
+
id: generateId(),
|
|
13587
|
+
content_hash: contentHash,
|
|
13588
|
+
size_bytes: input.data.length,
|
|
13589
|
+
created_at: nowIso2()
|
|
13590
|
+
};
|
|
13591
|
+
await this.store.put("attachment_blob", blob);
|
|
13592
|
+
}
|
|
13593
|
+
const attachment = {
|
|
13594
|
+
id: generateId(),
|
|
13595
|
+
note_id: input.noteId,
|
|
13596
|
+
blob_id: blob.id,
|
|
13597
|
+
document_type_id: null,
|
|
13598
|
+
mime_type: input.mimeType ?? null,
|
|
13599
|
+
extracted_text: input.extractedText ?? null,
|
|
13600
|
+
filename: input.filename,
|
|
13601
|
+
display_name: input.displayName ?? null,
|
|
13602
|
+
position: 0,
|
|
13603
|
+
created_at: nowIso2(),
|
|
13604
|
+
deleted_at: null
|
|
13605
|
+
};
|
|
13606
|
+
await this.store.put("attachment", attachment);
|
|
13607
|
+
return attachment;
|
|
12207
13608
|
}
|
|
12208
|
-
|
|
12209
|
-
|
|
13609
|
+
async get(id) {
|
|
13610
|
+
const attachment = await this.store.get("attachment", id);
|
|
13611
|
+
if (!attachment) throw new Error(`Attachment not found: ${id}`);
|
|
13612
|
+
return attachment;
|
|
13613
|
+
}
|
|
13614
|
+
/** Null when bytes are absent — the recoverable reference-only state. */
|
|
13615
|
+
async getBlob(attachmentId) {
|
|
13616
|
+
const checksum = await this.checksumOf(attachmentId);
|
|
13617
|
+
return checksum === null ? null : this.blobStore.read(checksum);
|
|
13618
|
+
}
|
|
13619
|
+
async hasBlob(attachmentId) {
|
|
13620
|
+
const checksum = await this.checksumOf(attachmentId);
|
|
13621
|
+
return checksum === null ? false : this.blobStore.has(checksum);
|
|
13622
|
+
}
|
|
13623
|
+
async list(noteId) {
|
|
13624
|
+
const items = (await this.store.list("attachment")).filter(
|
|
13625
|
+
(a) => a.note_id === noteId && a.deleted_at === null
|
|
13626
|
+
);
|
|
13627
|
+
items.sort(
|
|
13628
|
+
(a, b) => a.position - b.position || a.created_at.localeCompare(b.created_at)
|
|
13629
|
+
);
|
|
13630
|
+
return items;
|
|
13631
|
+
}
|
|
13632
|
+
/** Soft-delete the manifest; bytes are only swept via reconcile/gc. */
|
|
13633
|
+
async delete(id) {
|
|
13634
|
+
const attachment = await this.get(id);
|
|
13635
|
+
await this.store.put("attachment", { ...attachment, deleted_at: nowIso2() });
|
|
13636
|
+
}
|
|
13637
|
+
/** Authoritative live set: hashes referenced by non-deleted manifests. */
|
|
13638
|
+
async liveBlobChecksums() {
|
|
13639
|
+
const liveBlobIds = new Set(
|
|
13640
|
+
(await this.store.list("attachment")).filter((a) => a.deleted_at === null).map((a) => a.blob_id)
|
|
13641
|
+
);
|
|
13642
|
+
return (await this.store.list("attachment_blob")).filter((b) => liveBlobIds.has(b.id)).map((b) => b.content_hash);
|
|
13643
|
+
}
|
|
13644
|
+
/** Startup / post-quota reconciliation against canonical manifests (ADR-013 D4). */
|
|
13645
|
+
async reconcileBlobs(opts) {
|
|
13646
|
+
return this.blobStore.reconcile(await this.liveBlobChecksums(), opts);
|
|
13647
|
+
}
|
|
13648
|
+
/** Deferred reachability-based blob GC. */
|
|
13649
|
+
async gcBlobs(opts) {
|
|
13650
|
+
await this.reconcileBlobs();
|
|
13651
|
+
return this.blobStore.gc(opts);
|
|
13652
|
+
}
|
|
13653
|
+
async checksumOf(attachmentId) {
|
|
13654
|
+
const attachment = await this.get(attachmentId);
|
|
13655
|
+
const blob = await this.store.get("attachment_blob", attachment.blob_id);
|
|
13656
|
+
return blob?.content_hash ?? null;
|
|
13657
|
+
}
|
|
13658
|
+
};
|
|
13659
|
+
|
|
13660
|
+
// src/records/attachment-projection.ts
|
|
13661
|
+
async function projectAttachments(db, store) {
|
|
13662
|
+
const blobs = await store.list("attachment_blob");
|
|
13663
|
+
const attachments = await store.list("attachment");
|
|
13664
|
+
for (const blob of blobs) {
|
|
13665
|
+
await db.query(
|
|
13666
|
+
`INSERT INTO attachment_blob (id, content_hash, size_bytes, created_at)
|
|
13667
|
+
VALUES ($1, $2, $3, $4)
|
|
13668
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
13669
|
+
content_hash = EXCLUDED.content_hash,
|
|
13670
|
+
size_bytes = EXCLUDED.size_bytes`,
|
|
13671
|
+
[blob.id, blob.content_hash, blob.size_bytes, blob.created_at]
|
|
13672
|
+
);
|
|
13673
|
+
}
|
|
13674
|
+
for (const att of attachments) {
|
|
13675
|
+
await db.query(
|
|
13676
|
+
`INSERT INTO attachment (
|
|
13677
|
+
id, note_id, blob_id, document_type_id, mime_type, extracted_text,
|
|
13678
|
+
filename, display_name, position, status, created_at, deleted_at
|
|
13679
|
+
)
|
|
13680
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
|
13681
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
13682
|
+
mime_type = EXCLUDED.mime_type,
|
|
13683
|
+
extracted_text = EXCLUDED.extracted_text,
|
|
13684
|
+
filename = EXCLUDED.filename,
|
|
13685
|
+
display_name = EXCLUDED.display_name,
|
|
13686
|
+
position = EXCLUDED.position,
|
|
13687
|
+
status = EXCLUDED.status,
|
|
13688
|
+
deleted_at = EXCLUDED.deleted_at`,
|
|
13689
|
+
[
|
|
13690
|
+
att.id,
|
|
13691
|
+
att.note_id,
|
|
13692
|
+
att.blob_id,
|
|
13693
|
+
att.document_type_id,
|
|
13694
|
+
att.mime_type,
|
|
13695
|
+
att.extracted_text,
|
|
13696
|
+
att.filename,
|
|
13697
|
+
att.display_name,
|
|
13698
|
+
att.position,
|
|
13699
|
+
// Browser attach-time extraction: text present == processing done.
|
|
13700
|
+
att.extracted_text !== null && att.extracted_text !== "" ? "completed" : "uploaded",
|
|
13701
|
+
att.created_at,
|
|
13702
|
+
att.deleted_at
|
|
13703
|
+
]
|
|
13704
|
+
);
|
|
13705
|
+
}
|
|
13706
|
+
await db.query(
|
|
13707
|
+
`UPDATE attachment_blob ab
|
|
13708
|
+
SET reference_count = (
|
|
13709
|
+
SELECT COUNT(*) FROM attachment a
|
|
13710
|
+
WHERE a.blob_id = ab.id AND a.deleted_at IS NULL
|
|
13711
|
+
)`
|
|
13712
|
+
);
|
|
13713
|
+
return { blobs: blobs.length, attachments: attachments.length };
|
|
13714
|
+
}
|
|
13715
|
+
async function dropAttachmentProjection(db) {
|
|
13716
|
+
await db.query(`DELETE FROM attachment_embedding`);
|
|
13717
|
+
await db.query(`DELETE FROM attachment`);
|
|
13718
|
+
await db.query(`DELETE FROM attachment_blob`);
|
|
12210
13719
|
}
|
|
12211
13720
|
|
|
12212
13721
|
// src/index.ts
|
|
12213
|
-
var VERSION = "2026.7.
|
|
13722
|
+
var VERSION = "2026.7.7";
|
|
12214
13723
|
|
|
12215
|
-
export { AIWG_SCAN_REQUIRED_FIELDS, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, SHARD_FORMAT, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateFortemiCompatibilityResponse, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifySri };
|
|
13724
|
+
export { AIWG_SCAN_REQUIRED_FIELDS, AllowlistTrustStore, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CanonicalAttachmentsRepository, CanonicalNotesRepository, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, IdbRecordStore, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MemoryRecordStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, RECORD_COLLECTIONS, RECORD_SCHEMA_VERSION, RECORD_STORE_CAPABILITIES, SHARD_FORMAT, SIGNATURE_ENTRY, SIGNING_ENVELOPE_VERSION, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexFromKnowledgeShard, aiwgFortemiIndexToCommunityGraph, aiwgFortemiIndexToKnowledgeShard, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLazyBlobStore, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRecordStore, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dropAttachmentProjection, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, isShardSigningSupported, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, migrateLegacyBlobStore, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, projectAttachments, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, sidecarBlobDigests, signShard, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateFortemiCompatibilityResponse, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifyShardSignature, verifySri };
|
|
12216
13725
|
//# sourceMappingURL=index.js.map
|
|
12217
13726
|
//# sourceMappingURL=index.js.map
|