@fortemi/core 2026.7.3 → 2026.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,10 +1,10 @@
1
1
  import { v7 } from 'uuid';
2
- import { PGlite } from '@electric-sql/pglite';
3
- import { vector } from '@electric-sql/pglite/vector';
4
2
  import { sha256 } from '@noble/hashes/sha256';
3
+ import { blake3 } from '@noble/hashes/blake3';
5
4
  import { bytesToHex } from '@noble/hashes/utils';
6
5
  import { z } from 'zod';
7
6
  import { gzipSync, gunzipSync } from 'fflate';
7
+ import Ajv2020 from 'ajv/dist/2020.js';
8
8
 
9
9
  // src/uuid.ts
10
10
  function generateId() {
@@ -125,6 +125,8 @@ var TypedEventBus = class {
125
125
  this.wildcardListeners.clear();
126
126
  }
127
127
  };
128
+
129
+ // src/db.ts
128
130
  function getDataDir(persistence, archiveName) {
129
131
  switch (persistence) {
130
132
  case "opfs":
@@ -137,6 +139,10 @@ function getDataDir(persistence, archiveName) {
137
139
  }
138
140
  async function createPGliteInstance(persistence, archiveName = "default", options = {}) {
139
141
  const dataDir = getDataDir(persistence, archiveName);
142
+ const [{ PGlite }, { vector }] = await Promise.all([
143
+ import('@electric-sql/pglite'),
144
+ import('@electric-sql/pglite/vector')
145
+ ]);
140
146
  const pgliteOptions = {
141
147
  database: "postgres",
142
148
  // PGlite 0.4.x breaking change: explicit required
@@ -609,6 +615,134 @@ var migration0010 = {
609
615
  `
610
616
  };
611
617
 
618
+ // src/migrations/0011_embedding_member_metadata.ts
619
+ var migration0011 = {
620
+ version: 11,
621
+ name: "0011_embedding_member_metadata",
622
+ sql: `
623
+ ALTER TABLE embedding_set_member
624
+ ALTER COLUMN embedding_id DROP NOT NULL;
625
+
626
+ ALTER TABLE embedding_set_member
627
+ ADD COLUMN IF NOT EXISTS membership_type TEXT NOT NULL DEFAULT 'materialized';
628
+
629
+ ALTER TABLE embedding_set_member
630
+ ADD COLUMN IF NOT EXISTS added_at TIMESTAMPTZ NOT NULL DEFAULT now();
631
+
632
+ ALTER TABLE embedding_set_member
633
+ ADD COLUMN IF NOT EXISTS added_by TEXT;
634
+ `
635
+ };
636
+
637
+ // src/migrations/0012_embedding_configs.ts
638
+ var migration0012 = {
639
+ version: 12,
640
+ name: "0012_embedding_configs",
641
+ sql: `
642
+ CREATE TABLE IF NOT EXISTS embedding_config (
643
+ id TEXT PRIMARY KEY,
644
+ name TEXT NOT NULL,
645
+ description TEXT,
646
+ model TEXT NOT NULL,
647
+ dimension INTEGER NOT NULL,
648
+ chunk_size INTEGER NOT NULL,
649
+ chunk_overlap INTEGER NOT NULL,
650
+ is_default BOOLEAN NOT NULL DEFAULT false
651
+ );
652
+
653
+ CREATE INDEX IF NOT EXISTS idx_embedding_config_default ON embedding_config(is_default);
654
+ `
655
+ };
656
+
657
+ // src/migrations/0013_templates.ts
658
+ var migration0013 = {
659
+ version: 13,
660
+ name: "0013_templates",
661
+ sql: `
662
+ CREATE TABLE IF NOT EXISTS template (
663
+ id TEXT PRIMARY KEY,
664
+ name TEXT NOT NULL,
665
+ description TEXT,
666
+ content TEXT NOT NULL,
667
+ format TEXT NOT NULL DEFAULT 'markdown',
668
+ default_tags JSONB NOT NULL DEFAULT '[]',
669
+ collection_id TEXT,
670
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
671
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
672
+ );
673
+
674
+ CREATE INDEX IF NOT EXISTS idx_template_collection ON template(collection_id);
675
+ `
676
+ };
677
+
678
+ // src/migrations/0014_url_links.ts
679
+ var migration0014 = {
680
+ version: 14,
681
+ name: "0014_url_links",
682
+ sql: `
683
+ CREATE TABLE IF NOT EXISTS link_url_target (
684
+ id TEXT PRIMARY KEY,
685
+ source_note_id TEXT NOT NULL,
686
+ to_url TEXT NOT NULL,
687
+ link_type TEXT NOT NULL DEFAULT 'reference',
688
+ confidence REAL,
689
+ metadata_json JSONB,
690
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
691
+ updated_at TIMESTAMPTZ,
692
+ deleted_at TIMESTAMPTZ
693
+ );
694
+
695
+ CREATE INDEX IF NOT EXISTS idx_link_url_source ON link_url_target(source_note_id);
696
+ `
697
+ };
698
+
699
+ // src/migrations/0015_embedding_set_server_metadata.ts
700
+ var migration0015 = {
701
+ version: 15,
702
+ name: "0015_embedding_set_server_metadata",
703
+ sql: `
704
+ ALTER TABLE embedding_set
705
+ ADD COLUMN IF NOT EXISTS slug TEXT;
706
+
707
+ ALTER TABLE embedding_set
708
+ ADD COLUMN IF NOT EXISTS description TEXT;
709
+
710
+ ALTER TABLE embedding_set
711
+ ADD COLUMN IF NOT EXISTS document_count INTEGER;
712
+
713
+ ALTER TABLE embedding_set
714
+ ADD COLUMN IF NOT EXISTS embedding_count INTEGER;
715
+
716
+ ALTER TABLE embedding_set
717
+ ADD COLUMN IF NOT EXISTS is_system BOOLEAN NOT NULL DEFAULT FALSE;
718
+
719
+ ALTER TABLE embedding_set
720
+ ADD COLUMN IF NOT EXISTS keywords_json JSONB;
721
+ `
722
+ };
723
+
724
+ // src/migrations/0016_embedding_server_metadata.ts
725
+ var migration0016 = {
726
+ version: 16,
727
+ name: "0016_embedding_server_metadata",
728
+ sql: `
729
+ ALTER TABLE embedding
730
+ ADD COLUMN IF NOT EXISTS chunk_index INTEGER NOT NULL DEFAULT 0;
731
+
732
+ ALTER TABLE embedding
733
+ ADD COLUMN IF NOT EXISTS text TEXT NOT NULL DEFAULT '';
734
+
735
+ ALTER TABLE embedding
736
+ ADD COLUMN IF NOT EXISTS model TEXT;
737
+
738
+ ALTER TABLE embedding
739
+ DROP CONSTRAINT IF EXISTS embedding_note_id_embedding_set_id_key;
740
+
741
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_embedding_note_set_chunk
742
+ ON embedding(note_id, embedding_set_id, chunk_index);
743
+ `
744
+ };
745
+
612
746
  // src/migrations/index.ts
613
747
  var allMigrations = [
614
748
  migration0001,
@@ -620,7 +754,13 @@ var allMigrations = [
620
754
  migration0007,
621
755
  migration0008,
622
756
  migration0009,
623
- migration0010
757
+ migration0010,
758
+ migration0011,
759
+ migration0012,
760
+ migration0013,
761
+ migration0014,
762
+ migration0015,
763
+ migration0016
624
764
  ];
625
765
 
626
766
  // src/data-archive.ts
@@ -730,6 +870,9 @@ function computeHash(data) {
730
870
  const digest = sha256(data);
731
871
  return `sha256:${bytesToHex(digest)}`;
732
872
  }
873
+ function computeBlobHash(data) {
874
+ return `blake3:${bytesToHex(blake3(data))}`;
875
+ }
733
876
 
734
877
  // src/repositories/notes-repository.ts
735
878
  var NotesRepository = class {
@@ -1288,11 +1431,11 @@ var EmbeddingSetsRepository = class {
1288
1431
  [input.note_id, input.embedding_set_id]
1289
1432
  );
1290
1433
  const embeddingId = input.id ?? generateId();
1291
- const vector2 = `[${input.vector.join(",")}]`;
1434
+ const vector = `[${input.vector.join(",")}]`;
1292
1435
  await this.db.query(
1293
1436
  `INSERT INTO embedding (id, note_id, embedding_set_id, vector)
1294
1437
  VALUES ($1, $2, $3, $4::vector)`,
1295
- [embeddingId, input.note_id, input.embedding_set_id, vector2]
1438
+ [embeddingId, input.note_id, input.embedding_set_id, vector]
1296
1439
  );
1297
1440
  await this.db.query(
1298
1441
  `INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
@@ -1454,7 +1597,7 @@ var EmbeddingSetsRepository = class {
1454
1597
  if (criteria.conceptIds && criteria.conceptIds.length > 0) {
1455
1598
  throw new Error("Unsupported virtual embedding-set criteria field: conceptIds");
1456
1599
  }
1457
- const conditions = ["e.embedding_set_id = $1"];
1600
+ const conditions = ["e.embedding_set_id = $1", "n.deleted_at IS NULL"];
1458
1601
  const params = [source.baseSetId];
1459
1602
  let idx = 2;
1460
1603
  if (criteria.noteIds?.length) {
@@ -1657,7 +1800,7 @@ var ATTACHMENT_TEXT_JOIN2 = `
1657
1800
  WHERE deleted_at IS NULL
1658
1801
  GROUP BY note_id
1659
1802
  ) ax ON ax.note_id = n.id`;
1660
- var COMBINED_TEXT_SQL = `(coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, ''))`;
1803
+ var COMBINED_TEXT_SQL = `trim(both from (coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, '')))`;
1661
1804
  var COMBINED_TEXT_VECTOR_SQL2 = `to_tsvector('english', ${COMBINED_TEXT_SQL})`;
1662
1805
  var SearchRepository = class {
1663
1806
  constructor(db, semanticAvailable = false) {
@@ -2285,11 +2428,25 @@ function linkToBackend(link) {
2285
2428
  createdAt: toIso(link.created_at)
2286
2429
  };
2287
2430
  }
2431
+ function urlLinkToBackend(link) {
2432
+ const metadata = typeof link.metadata_json === "string" ? JSON.parse(link.metadata_json) : link.metadata_json ?? void 0;
2433
+ return {
2434
+ id: link.id,
2435
+ fromNoteId: link.source_note_id,
2436
+ toNoteId: null,
2437
+ toUrl: link.to_url,
2438
+ kind: link.link_type,
2439
+ score: link.confidence,
2440
+ createdAt: toIso(link.created_at),
2441
+ ...metadata ? { metadata } : {}
2442
+ };
2443
+ }
2288
2444
  function shardLinkToBackend(link) {
2289
2445
  return {
2290
2446
  id: link.id,
2291
2447
  fromNoteId: link.from_note_id,
2292
2448
  toNoteId: link.to_note_id,
2449
+ toUrl: link.to_url,
2293
2450
  kind: link.kind,
2294
2451
  score: link.score,
2295
2452
  createdAt: link.created_at,
@@ -2335,7 +2492,8 @@ function remoteLinkToBackend(link) {
2335
2492
  return {
2336
2493
  id: link.id,
2337
2494
  fromNoteId: link.fromNoteId ?? link.from_note_id ?? link.source_note_id ?? "",
2338
- toNoteId: link.toNoteId ?? link.to_note_id ?? link.target_note_id ?? "",
2495
+ toNoteId: link.toNoteId ?? link.to_note_id ?? link.target_note_id ?? null,
2496
+ toUrl: link.toUrl ?? link.to_url ?? null,
2339
2497
  kind: link.kind ?? link.link_type ?? "",
2340
2498
  score: link.score ?? link.confidence ?? null,
2341
2499
  createdAt: link.createdAt ?? link.created_at ?? "",
@@ -2357,7 +2515,15 @@ function createPGliteBackend(db, options = {}) {
2357
2515
  const links = new LinksRepository(db);
2358
2516
  async function linksOf(id) {
2359
2517
  const result = await links.listForNote(id);
2360
- return [...result.outbound, ...result.inbound].map(linkToBackend);
2518
+ const urlLinks = await db.query(
2519
+ `SELECT * FROM link_url_target WHERE source_note_id = $1 AND deleted_at IS NULL ORDER BY created_at`,
2520
+ [id]
2521
+ );
2522
+ return [
2523
+ ...result.outbound.map(linkToBackend),
2524
+ ...result.inbound.map(linkToBackend),
2525
+ ...urlLinks.rows.map(urlLinkToBackend)
2526
+ ];
2361
2527
  }
2362
2528
  async function conceptsOf(id) {
2363
2529
  const result = await db.query(
@@ -2988,6 +3154,20 @@ var MigrationRunner = class {
2988
3154
 
2989
3155
  // src/archive-manager.ts
2990
3156
  var ArchiveManager = class {
3157
+ /**
3158
+ * Persistence is a PLUGGABLE, opt-in backend (issue #261). Pass a
3159
+ * `StorageBackendFactory` to run against any `StorageBackend`
3160
+ * implementation; PGlite is merely the built-in default, selected only when
3161
+ * the convenience `PersistenceMode` string form is used. Because
3162
+ * `defaultStorageBackendFactory` reaches PGlite through the lazily-imported
3163
+ * `createPGliteInstance` (see `db.ts`), a consumer that supplies its own
3164
+ * factory — or never opens an archive at all — never pulls the PGlite WASM
3165
+ * engine into its bundle.
3166
+ *
3167
+ * @param persistenceOrFactory `PersistenceMode` string (uses the built-in
3168
+ * PGlite backend) OR a custom `StorageBackendFactory`.
3169
+ * @param persistenceOverride persistence hint passed to a custom factory.
3170
+ */
2991
3171
  constructor(persistenceOrFactory, events, persistenceOverride) {
2992
3172
  this.events = events;
2993
3173
  if (typeof persistenceOrFactory === "string") {
@@ -3096,6 +3276,205 @@ function createFortemi(config) {
3096
3276
  };
3097
3277
  }
3098
3278
 
3279
+ // src/server-compatibility.ts
3280
+ var FORTEMI_COMPATIBILITY_PATH = "/api/v1/system/compatibility";
3281
+ var FORTEMI_COMPATIBILITY_STATES = [
3282
+ "available",
3283
+ "degraded",
3284
+ "preview",
3285
+ "unavailable",
3286
+ "unknown"
3287
+ ];
3288
+ var FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES = [
3289
+ "core_notes",
3290
+ "search",
3291
+ "jobs",
3292
+ "realtime_activity",
3293
+ "hosted_auth",
3294
+ "premium_components",
3295
+ "backoffice_api",
3296
+ "audit_posture",
3297
+ "quota_status",
3298
+ "kms_status",
3299
+ "mcp_scope_gate"
3300
+ ];
3301
+ function isRecord(value) {
3302
+ return !!value && typeof value === "object" && !Array.isArray(value);
3303
+ }
3304
+ function isSupportedState(value) {
3305
+ return typeof value === "string" && FORTEMI_COMPATIBILITY_STATES.includes(value);
3306
+ }
3307
+ function requireRecord(parent, key, errors) {
3308
+ const value = parent[key];
3309
+ if (!isRecord(value)) {
3310
+ errors.push(`${key} must be an object`);
3311
+ return null;
3312
+ }
3313
+ return value;
3314
+ }
3315
+ function requireString(parent, key, path, errors) {
3316
+ const value = parent[key];
3317
+ if (typeof value !== "string" || value.length === 0) {
3318
+ errors.push(`${path}.${key} must be a non-empty string`);
3319
+ return null;
3320
+ }
3321
+ return value;
3322
+ }
3323
+ function requireBoolean(parent, key, path, errors) {
3324
+ const value = parent[key];
3325
+ if (typeof value !== "boolean") {
3326
+ errors.push(`${path}.${key} must be a boolean`);
3327
+ return null;
3328
+ }
3329
+ return value;
3330
+ }
3331
+ function fortemiCompatibilityUrl(baseUrl = "http://localhost:3000") {
3332
+ const trimmed = baseUrl.trim().replace(/\/+$/, "");
3333
+ if (trimmed.endsWith(FORTEMI_COMPATIBILITY_PATH)) return trimmed;
3334
+ return `${trimmed}${FORTEMI_COMPATIBILITY_PATH}`;
3335
+ }
3336
+ function validateFortemiCompatibilityResponse(raw) {
3337
+ const errors = [];
3338
+ const warnings = [];
3339
+ if (!isRecord(raw)) {
3340
+ return {
3341
+ ok: false,
3342
+ errors: ["response must be a JSON object"],
3343
+ warnings
3344
+ };
3345
+ }
3346
+ if (raw.schema_version !== 1) {
3347
+ errors.push("schema_version must be 1");
3348
+ }
3349
+ requireString(raw, "contract_revision", "response", errors);
3350
+ const api = requireRecord(raw, "api", errors);
3351
+ if (api) {
3352
+ const name = requireString(api, "name", "api", errors);
3353
+ if (name && name !== "fortemi") {
3354
+ errors.push("api.name must be fortemi");
3355
+ }
3356
+ requireString(api, "version", "api", errors);
3357
+ requireString(api, "minimum_hotm_enterprise_client", "api", errors);
3358
+ requireBoolean(api, "git_sha_present", "api", errors);
3359
+ requireBoolean(api, "build_date_present", "api", errors);
3360
+ }
3361
+ const deployment = requireRecord(raw, "deployment", errors);
3362
+ if (deployment) {
3363
+ requireString(deployment, "mode", "deployment", errors);
3364
+ requireString(deployment, "edition", "deployment", errors);
3365
+ requireBoolean(deployment, "hosted_multi_tenant_ready", "deployment", errors);
3366
+ }
3367
+ const auth = requireRecord(raw, "auth", errors);
3368
+ if (auth) {
3369
+ requireBoolean(auth, "required", "auth", errors);
3370
+ requireString(auth, "mode", "auth", errors);
3371
+ requireBoolean(auth, "oauth_issuer_configured", "auth", errors);
3372
+ requireBoolean(auth, "tenant_context_available", "auth", errors);
3373
+ }
3374
+ const capabilities = requireRecord(raw, "capabilities", errors);
3375
+ if (capabilities) {
3376
+ for (const key of FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES) {
3377
+ if (!isRecord(capabilities[key])) {
3378
+ errors.push(`capabilities.${key} must be present as an object`);
3379
+ }
3380
+ }
3381
+ for (const [key, value] of Object.entries(capabilities)) {
3382
+ if (!isRecord(value)) {
3383
+ errors.push(`capabilities.${key} must be an object`);
3384
+ continue;
3385
+ }
3386
+ if (!isSupportedState(value.state)) {
3387
+ errors.push(`capabilities.${key}.state must be one of ${FORTEMI_COMPATIBILITY_STATES.join(", ")}`);
3388
+ }
3389
+ if ("reason_code" in value && typeof value.reason_code !== "string") {
3390
+ errors.push(`capabilities.${key}.reason_code must be a string when present`);
3391
+ }
3392
+ if (value.state !== "available" && !value.reason_code) {
3393
+ warnings.push(`capabilities.${key} is ${String(value.state)} without reason_code`);
3394
+ }
3395
+ }
3396
+ }
3397
+ const links = requireRecord(raw, "links", errors);
3398
+ if (links) {
3399
+ requireString(links, "openapi", "links", errors);
3400
+ requireString(links, "asyncapi", "links", errors);
3401
+ requireString(links, "health", "links", errors);
3402
+ requireString(links, "streaming_health", "links", errors);
3403
+ }
3404
+ return {
3405
+ ok: errors.length === 0,
3406
+ errors,
3407
+ warnings,
3408
+ response: errors.length === 0 ? raw : void 0
3409
+ };
3410
+ }
3411
+ function formatFortemiCompatibilitySummary(response) {
3412
+ const capabilitySummary = Object.entries(response.capabilities).map(([key, value]) => `${key}=${value.state}`).sort().join(", ");
3413
+ return [
3414
+ `Fortemi compatibility ${response.contract_revision}`,
3415
+ `api=${response.api.name}@${response.api.version}`,
3416
+ `deployment=${response.deployment.mode}/${response.deployment.edition}`,
3417
+ `auth=${response.auth.mode}`,
3418
+ `capabilities: ${capabilitySummary}`
3419
+ ].join("\n");
3420
+ }
3421
+ async function fetchAndValidateFortemiCompatibility(options = {}) {
3422
+ const fetchImpl = options.fetchImpl ?? globalThis.fetch;
3423
+ if (!fetchImpl) {
3424
+ return {
3425
+ ok: false,
3426
+ url: fortemiCompatibilityUrl(options.baseUrl),
3427
+ errors: ["fetch is not available in this runtime"],
3428
+ warnings: []
3429
+ };
3430
+ }
3431
+ const url = fortemiCompatibilityUrl(options.baseUrl);
3432
+ const controller = new AbortController();
3433
+ const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? 5e3);
3434
+ try {
3435
+ const response = await fetchImpl(url, {
3436
+ method: "GET",
3437
+ headers: { accept: "application/json" },
3438
+ signal: controller.signal
3439
+ });
3440
+ const status = response.status;
3441
+ const contentType = response.headers.get("content-type") ?? "";
3442
+ if (!response.ok) {
3443
+ return {
3444
+ ok: false,
3445
+ url,
3446
+ status,
3447
+ errors: [`Fortemi compatibility endpoint returned HTTP ${status}`],
3448
+ warnings: []
3449
+ };
3450
+ }
3451
+ if (!contentType.toLowerCase().includes("application/json")) {
3452
+ return {
3453
+ ok: false,
3454
+ url,
3455
+ status,
3456
+ errors: [`Fortemi compatibility endpoint must return JSON; got ${contentType || "unknown content-type"}`],
3457
+ warnings: []
3458
+ };
3459
+ }
3460
+ return {
3461
+ ...validateFortemiCompatibilityResponse(await response.json()),
3462
+ url,
3463
+ status
3464
+ };
3465
+ } catch (error) {
3466
+ const message = error instanceof Error ? error.message : String(error);
3467
+ return {
3468
+ ok: false,
3469
+ url,
3470
+ errors: [`Fortemi compatibility endpoint is unreachable: ${message}`],
3471
+ warnings: []
3472
+ };
3473
+ } finally {
3474
+ clearTimeout(timeout);
3475
+ }
3476
+ }
3477
+
3099
3478
  // src/service-worker/register.ts
3100
3479
  async function registerServiceWorker(swUrl = "/sw.js") {
3101
3480
  if (!("serviceWorker" in navigator)) {
@@ -3862,6 +4241,7 @@ async function getNoteTextWithExtractedAttachments(db, noteId) {
3862
4241
  ' ORDER BY a.position, a.created_at)
3863
4242
  FILTER (WHERE a.extracted_text IS NOT NULL AND a.extracted_text <> ''), '') as extracted_text
3864
4243
  FROM note_revised_current c
4244
+ JOIN note n ON n.id = c.note_id AND n.deleted_at IS NULL
3865
4245
  LEFT JOIN attachment a ON a.note_id = c.note_id AND a.deleted_at IS NULL
3866
4246
  WHERE c.note_id = $1
3867
4247
  GROUP BY c.content`,
@@ -4159,7 +4539,7 @@ function conceptTaggingHandler(job, db) {
4159
4539
  const llmFn2 = getLlmFunction();
4160
4540
  if (!llmFn2) return { skipped: true, reason: "no LLM function registered" };
4161
4541
  const noteText = await getNoteTextWithExtractedAttachments(db, job.note_id);
4162
- if (!noteText) throw new Error(`No content found for note ${job.note_id}`);
4542
+ if (!noteText) return { skipped: true, reason: "note missing, deleted, or has no content" };
4163
4543
  const content = noteText.combined;
4164
4544
  const prompt = `Task: Extract 3-5 topic tags from the text below.
4165
4545
  Rules:
@@ -4197,20 +4577,25 @@ Tags:`;
4197
4577
  function linkingHandler(job, db) {
4198
4578
  return (async () => {
4199
4579
  const embResult = await db.query(
4200
- `SELECT id FROM embedding WHERE note_id = $1 LIMIT 1`,
4580
+ `SELECT e.id FROM embedding e
4581
+ JOIN note n ON n.id = e.note_id AND n.deleted_at IS NULL
4582
+ WHERE e.note_id = $1 LIMIT 1`,
4201
4583
  [job.note_id]
4202
4584
  );
4203
4585
  if (embResult.rows.length === 0) {
4204
4586
  return { skipped: true, reason: "no embeddings for this note yet" };
4205
4587
  }
4206
4588
  const vecResult = await db.query(
4207
- `SELECT vector::text FROM embedding WHERE note_id = $1 LIMIT 1`,
4589
+ `SELECT e.vector::text FROM embedding e
4590
+ JOIN note n ON n.id = e.note_id AND n.deleted_at IS NULL
4591
+ WHERE e.note_id = $1 LIMIT 1`,
4208
4592
  [job.note_id]
4209
4593
  );
4210
4594
  if (vecResult.rows.length === 0) return { skipped: true, reason: "no vector found" };
4211
4595
  const similar = await db.query(
4212
4596
  `SELECT e.note_id, e.vector <=> (SELECT vector FROM embedding WHERE note_id = $1 LIMIT 1) as distance
4213
4597
  FROM embedding e
4598
+ JOIN note n ON n.id = e.note_id AND n.deleted_at IS NULL
4214
4599
  WHERE e.note_id != $1
4215
4600
  ORDER BY distance ASC
4216
4601
  LIMIT 5`,
@@ -5027,14 +5412,16 @@ var AttachmentsRepository = class {
5027
5412
  /**
5028
5413
  * Attach a binary file to a note.
5029
5414
  *
5030
- * If a blob with the same SHA-256 content hash already exists, the existing
5415
+ * If a blob with the same BLAKE3 content hash already exists, the existing
5031
5416
  * blob row is reused (deduplication). Otherwise a new blob row is inserted
5032
- * and the raw bytes are written to the BlobStore.
5417
+ * and the raw bytes are written to the BlobStore. The BLAKE3 `content_hash`
5418
+ * (`blake3:<hex>`) matches the server convention and is the key used by the
5419
+ * portable Knowledge-Shard byte sidecar.
5033
5420
  *
5034
5421
  * Returns the newly created AttachmentRow.
5035
5422
  */
5036
5423
  async attach(input) {
5037
- const contentHash = computeHash(input.data);
5424
+ const contentHash = computeBlobHash(input.data);
5038
5425
  const sizeBytes = input.data.length;
5039
5426
  let blobId;
5040
5427
  const existing = await this.db.query(
@@ -5148,6 +5535,10 @@ async function manageAttachments(db, blobStore, rawInput) {
5148
5535
  extractedText: input.extracted_text,
5149
5536
  displayName: input.display_name
5150
5537
  });
5538
+ if (input.extracted_text) {
5539
+ await enqueueJob(db, { noteId: input.note_id, jobType: "embedding" });
5540
+ await enqueueJob(db, { noteId: input.note_id, jobType: "concept_tagging" });
5541
+ }
5151
5542
  return { action: "attach", attachment, size_bytes: data.length };
5152
5543
  }
5153
5544
  case "list": {
@@ -5171,7 +5562,12 @@ async function manageAttachments(db, blobStore, rawInput) {
5171
5562
  }
5172
5563
  case "delete": {
5173
5564
  if (!input.attachment_id) throw new Error("attachment_id required for delete");
5565
+ const attachment = await repo.get(input.attachment_id);
5174
5566
  await repo.delete(input.attachment_id);
5567
+ if (attachment.extracted_text) {
5568
+ await enqueueJob(db, { noteId: attachment.note_id, jobType: "embedding" });
5569
+ await enqueueJob(db, { noteId: attachment.note_id, jobType: "concept_tagging" });
5570
+ }
5175
5571
  return { action: "delete", attachment_id: input.attachment_id };
5176
5572
  }
5177
5573
  }
@@ -5398,17 +5794,17 @@ async function embeddingGenerationHandler(job, db) {
5398
5794
  const fn = embedFn;
5399
5795
  if (!fn) return { skipped: true, reason: "no embed function registered" };
5400
5796
  const noteText = await getNoteTextWithExtractedAttachments(db, job.note_id);
5401
- if (!noteText) throw new Error(`No content for note ${job.note_id}`);
5797
+ if (!noteText) return { skipped: true, reason: "note missing, deleted, or has no content" };
5402
5798
  const content = noteText.combined;
5403
5799
  const chunks = chunkText(content);
5404
5800
  const embeddings = await fn(chunks);
5405
- const vector2 = averageEmbeddings(embeddings);
5801
+ const vector = averageEmbeddings(embeddings);
5406
5802
  const embeddingSets = new EmbeddingSetsRepository(db);
5407
5803
  const set = await embeddingSets.ensureDefault();
5408
5804
  await embeddingSets.putEmbedding({
5409
5805
  note_id: job.note_id,
5410
5806
  embedding_set_id: set.id,
5411
- vector: vector2
5807
+ vector
5412
5808
  });
5413
5809
  return { chunks: chunks.length, embeddings: embeddings.length, setId: set.id };
5414
5810
  }
@@ -5827,13 +6223,13 @@ var OpenAICompatibleProvider = class {
5827
6223
  throw new Error("Streaming not supported: response body is null");
5828
6224
  }
5829
6225
  const reader = response.body.getReader();
5830
- const decoder4 = new TextDecoder();
6226
+ const decoder6 = new TextDecoder();
5831
6227
  let buffer = "";
5832
6228
  try {
5833
6229
  while (true) {
5834
6230
  const { done, value } = await reader.read();
5835
6231
  if (done) break;
5836
- buffer += decoder4.decode(value, { stream: true });
6232
+ buffer += decoder6.decode(value, { stream: true });
5837
6233
  const lines = buffer.split("\n");
5838
6234
  buffer = lines.pop() ?? "";
5839
6235
  for (const line of lines) {
@@ -6453,6 +6849,23 @@ function createCspReportHandler(onReport) {
6453
6849
  // src/shard/types.ts
6454
6850
  var CURRENT_SHARD_VERSION = "1.0.0";
6455
6851
  var SHARD_FORMAT = "matric-shard";
6852
+ function parseVersion(value) {
6853
+ return value.split(".").map((segment) => {
6854
+ const match = segment.match(/^\d+/);
6855
+ return match ? Number.parseInt(match[0], 10) : 0;
6856
+ });
6857
+ }
6858
+ function compareShardVersions(left, right) {
6859
+ const leftParts = parseVersion(left);
6860
+ const rightParts = parseVersion(right);
6861
+ const length = Math.max(leftParts.length, rightParts.length);
6862
+ for (let index = 0; index < length; index += 1) {
6863
+ const leftValue = leftParts[index] ?? 0;
6864
+ const rightValue = rightParts[index] ?? 0;
6865
+ if (leftValue !== rightValue) return leftValue > rightValue ? 1 : -1;
6866
+ }
6867
+ return 0;
6868
+ }
6456
6869
  var BLOCK_SIZE = 512;
6457
6870
  var USTAR_MAGIC = "ustar\x0000";
6458
6871
  function writeString(buf, offset, str, len) {
@@ -6542,8 +6955,14 @@ function decodeTar(tarData) {
6542
6955
  }
6543
6956
  return files;
6544
6957
  }
6545
- function packTarGz(files) {
6958
+ function packTarGz(files, opts) {
6546
6959
  const tarData = encodeTar(files);
6960
+ const cap = opts?.maxDecompressedBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES;
6961
+ if (tarData.byteLength > cap) {
6962
+ throw new Error(
6963
+ "Refusing to create archive: uncompressed size " + tarData.byteLength + " exceeds cap " + cap + " bytes"
6964
+ );
6965
+ }
6547
6966
  return gzipSync(tarData);
6548
6967
  }
6549
6968
  var DEFAULT_MAX_DECOMPRESSED_BYTES = 256 * 1024 * 1024;
@@ -6599,7 +7018,8 @@ function noteToShard(note) {
6599
7018
  title: note.title,
6600
7019
  original_content: note.original_content,
6601
7020
  revised_content: note.revised_content,
6602
- ...note.binary_sources?.length ? { binary_sources: note.binary_sources } : {},
7021
+ collection_id: note.collection_id ?? null,
7022
+ ...note.attachments?.length ? { attachments: note.attachments } : {},
6603
7023
  format: note.format,
6604
7024
  source: note.source,
6605
7025
  starred: note.is_starred,
@@ -6611,6 +7031,7 @@ function noteToShard(note) {
6611
7031
  };
6612
7032
  }
6613
7033
  function noteFromShard(shard) {
7034
+ const attachments = shard.attachments ?? shard.binary_sources;
6614
7035
  return {
6615
7036
  id: shard.id,
6616
7037
  title: shard.title,
@@ -6620,7 +7041,8 @@ function noteFromShard(shard) {
6620
7041
  is_archived: shard.archived,
6621
7042
  original_content: shard.original_content,
6622
7043
  revised_content: shard.revised_content,
6623
- binary_sources: shard.binary_sources,
7044
+ collection_id: shard.collection_id ?? null,
7045
+ attachments,
6624
7046
  tags: shard.tags,
6625
7047
  created_at: shard.created_at,
6626
7048
  updated_at: shard.updated_at,
@@ -6632,9 +7054,24 @@ function linkToShard(link) {
6632
7054
  id: link.id,
6633
7055
  from_note_id: link.source_note_id,
6634
7056
  to_note_id: link.target_note_id,
7057
+ to_url: null,
7058
+ kind: link.link_type,
7059
+ score: link.confidence,
7060
+ created_at: toISOString(link.created_at),
7061
+ metadata: null
7062
+ };
7063
+ }
7064
+ function urlLinkToShard(link) {
7065
+ const metadata = typeof link.metadata_json === "string" ? JSON.parse(link.metadata_json) : link.metadata_json ?? null;
7066
+ return {
7067
+ id: link.id,
7068
+ from_note_id: link.source_note_id,
7069
+ to_note_id: null,
7070
+ to_url: link.to_url,
6635
7071
  kind: link.link_type,
6636
7072
  score: link.confidence,
6637
- created_at: toISOString(link.created_at)
7073
+ created_at: toISOString(link.created_at),
7074
+ metadata
6638
7075
  };
6639
7076
  }
6640
7077
  function linkFromShard(shard) {
@@ -6642,9 +7079,11 @@ function linkFromShard(shard) {
6642
7079
  id: shard.id,
6643
7080
  source_note_id: shard.from_note_id,
6644
7081
  target_note_id: shard.to_note_id,
7082
+ to_url: shard.to_url,
6645
7083
  link_type: shard.kind,
6646
7084
  confidence: shard.score,
6647
- created_at: shard.created_at
7085
+ created_at: shard.created_at,
7086
+ metadata: shard.metadata
6648
7087
  };
6649
7088
  }
6650
7089
  function collectionToShard(collection, noteCount) {
@@ -6672,14 +7111,31 @@ function tagsToShard(allTags) {
6672
7111
  created_at: toISOString(t.created_at)
6673
7112
  }));
6674
7113
  }
6675
- function tagsFromShard(shardTags) {
6676
- return [...new Set(shardTags.map((t) => t.name))];
7114
+ function templateToShard(template) {
7115
+ return {
7116
+ id: template.id,
7117
+ name: template.name,
7118
+ description: template.description,
7119
+ content: template.content,
7120
+ format: template.format,
7121
+ default_tags: Array.isArray(template.default_tags) ? template.default_tags : JSON.parse(template.default_tags),
7122
+ collection_id: template.collection_id,
7123
+ created_at: toISOString(template.created_at),
7124
+ updated_at: toISOString(template.updated_at)
7125
+ };
6677
7126
  }
6678
7127
  function embeddingSetToShard(set) {
7128
+ const name = set.name ?? set.model_name;
6679
7129
  return {
6680
7130
  id: set.id,
6681
- name: set.name ?? set.model_name,
7131
+ name,
7132
+ slug: set.slug ?? slugifyEmbeddingSet(name),
7133
+ description: set.description ?? null,
6682
7134
  purpose: set.purpose ?? null,
7135
+ document_count: set.document_count ?? 0,
7136
+ embedding_count: set.embedding_count ?? 0,
7137
+ is_system: set.is_system ?? false,
7138
+ keywords: jsonStringArray(set.keywords_json),
6683
7139
  model: set.model_name,
6684
7140
  dimension: set.dimensions,
6685
7141
  kind: set.kind ?? "physical",
@@ -6694,11 +7150,17 @@ function embeddingSetToShard(set) {
6694
7150
  updated_at: set.updated_at ? toISOString(set.updated_at) : void 0
6695
7151
  };
6696
7152
  }
6697
- function embeddingSetFromShard(shard) {
7153
+ function embeddingSetFromShard(shard, fallbackCreatedAt) {
6698
7154
  return {
6699
7155
  id: shard.id,
6700
7156
  name: shard.name ?? shard.model,
7157
+ slug: shard.slug ?? null,
7158
+ description: shard.description ?? null,
6701
7159
  purpose: shard.purpose ?? null,
7160
+ document_count: shard.document_count ?? null,
7161
+ embedding_count: shard.embedding_count ?? null,
7162
+ is_system: shard.is_system ?? false,
7163
+ keywords_json: jsonString(shard.keywords ?? []),
6702
7164
  model_name: shard.model,
6703
7165
  dimensions: shard.dimension,
6704
7166
  kind: shard.kind ?? "physical",
@@ -6709,7 +7171,7 @@ function embeddingSetFromShard(shard) {
6709
7171
  compatibility_json: jsonString(shard.compatibility),
6710
7172
  materialization_json: jsonString(shard.materialization),
6711
7173
  freshness_json: jsonString(shard.freshness),
6712
- created_at: shard.created_at,
7174
+ created_at: shard.created_at ?? fallbackCreatedAt,
6713
7175
  updated_at: shard.updated_at ?? null
6714
7176
  };
6715
7177
  }
@@ -6717,15 +7179,32 @@ function embeddingSetMemberToShard(member) {
6717
7179
  return {
6718
7180
  embedding_set_id: member.embedding_set_id,
6719
7181
  note_id: member.note_id,
6720
- embedding_id: member.embedding_id
7182
+ membership_type: member.membership_type ?? "materialized",
7183
+ added_at: toISOString(member.added_at ?? /* @__PURE__ */ new Date()),
7184
+ added_by: member.added_by ?? null
7185
+ };
7186
+ }
7187
+ function embeddingConfigToShard(config) {
7188
+ return {
7189
+ id: config.id,
7190
+ name: config.name,
7191
+ description: config.description ?? null,
7192
+ model: config.model,
7193
+ dimension: config.dimension,
7194
+ chunk_size: config.chunk_size,
7195
+ chunk_overlap: config.chunk_overlap,
7196
+ is_default: config.is_default
6721
7197
  };
6722
7198
  }
6723
7199
  function embeddingToShard(emb) {
6724
7200
  return {
6725
7201
  id: emb.id,
6726
7202
  note_id: emb.note_id,
6727
- embedding_set_id: emb.embedding_set_id,
7203
+ chunk_index: emb.chunk_index ?? 0,
7204
+ text: emb.text ?? "",
6728
7205
  vector: typeof emb.vector === "string" ? parseVector2(emb.vector) : emb.vector,
7206
+ model: emb.model ?? emb.model_name ?? "unknown",
7207
+ embedding_set_id: emb.embedding_set_id,
6729
7208
  created_at: toISOString(emb.created_at)
6730
7209
  };
6731
7210
  }
@@ -6733,9 +7212,12 @@ function embeddingFromShard(shard) {
6733
7212
  return {
6734
7213
  id: shard.id,
6735
7214
  note_id: shard.note_id,
6736
- embedding_set_id: shard.embedding_set_id,
7215
+ embedding_set_id: shard.embedding_set_id ?? null,
7216
+ chunk_index: shard.chunk_index,
7217
+ text: shard.text,
6737
7218
  vector: `[${shard.vector.join(",")}]`,
6738
- created_at: shard.created_at
7219
+ model: shard.model,
7220
+ created_at: shard.created_at ?? null
6739
7221
  };
6740
7222
  }
6741
7223
  function skosSchemeToShard(scheme) {
@@ -6807,6 +7289,15 @@ function parseJsonObjectField(value) {
6807
7289
  const parsed = JSON.parse(value);
6808
7290
  return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
6809
7291
  }
7292
+ function jsonStringArray(value) {
7293
+ if (value == null) return [];
7294
+ if (Array.isArray(value)) return value.map(String);
7295
+ if (typeof value === "string") {
7296
+ const parsed = JSON.parse(value);
7297
+ return Array.isArray(parsed) ? parsed.map(String) : [];
7298
+ }
7299
+ return [];
7300
+ }
6810
7301
  function jsonObject2(value) {
6811
7302
  if (value == null) return null;
6812
7303
  if (typeof value === "string") return JSON.parse(value);
@@ -6815,6 +7306,34 @@ function jsonObject2(value) {
6815
7306
  function jsonString(value) {
6816
7307
  return value == null ? null : JSON.stringify(value);
6817
7308
  }
7309
+ function slugifyEmbeddingSet(value) {
7310
+ const slug = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
7311
+ return slug || "embedding-set";
7312
+ }
7313
+
7314
+ // src/shard/blob-sidecar.ts
7315
+ var SIDECAR_PREFIX = "blobs/";
7316
+ function blobChecksumToHex(checksum) {
7317
+ const sep = checksum.indexOf(":");
7318
+ return sep >= 0 ? checksum.slice(sep + 1) : checksum;
7319
+ }
7320
+ function sidecarEntryName(checksum) {
7321
+ return SIDECAR_PREFIX + blobChecksumToHex(checksum);
7322
+ }
7323
+ function isSidecarEntry(name) {
7324
+ if (!name.startsWith(SIDECAR_PREFIX)) return false;
7325
+ const rest = name.slice(SIDECAR_PREFIX.length);
7326
+ return rest.length > 0 && !rest.includes("/");
7327
+ }
7328
+ function collectSidecarBlobs(files) {
7329
+ const blobs = /* @__PURE__ */ new Map();
7330
+ for (const [name, bytes] of files) {
7331
+ if (isSidecarEntry(name)) {
7332
+ blobs.set(name.slice(SIDECAR_PREFIX.length), bytes);
7333
+ }
7334
+ }
7335
+ return blobs;
7336
+ }
6818
7337
 
6819
7338
  // src/shard/shard-export.ts
6820
7339
  var encoder = new TextEncoder();
@@ -6836,7 +7355,8 @@ async function exportShard(db, options) {
6836
7355
  noteQuery = `SELECT n.id, n.title, n.format, n.source, n.is_starred, n.is_archived,
6837
7356
  n.created_at, n.updated_at, n.deleted_at,
6838
7357
  o.content as original_content,
6839
- c.content as revised_content
7358
+ c.content as revised_content,
7359
+ $1::text as collection_id
6840
7360
  FROM note n
6841
7361
  LEFT JOIN note_original o ON o.note_id = n.id
6842
7362
  LEFT JOIN note_revised_current c ON c.note_id = n.id
@@ -6848,7 +7368,14 @@ async function exportShard(db, options) {
6848
7368
  noteQuery = `SELECT n.id, n.title, n.format, n.source, n.is_starred, n.is_archived,
6849
7369
  n.created_at, n.updated_at, n.deleted_at,
6850
7370
  o.content as original_content,
6851
- c.content as revised_content
7371
+ c.content as revised_content,
7372
+ (
7373
+ SELECT cn.collection_id
7374
+ FROM collection_note cn
7375
+ WHERE cn.note_id = n.id
7376
+ ORDER BY cn.position, cn.added_at
7377
+ LIMIT 1
7378
+ ) as collection_id
6852
7379
  FROM note n
6853
7380
  LEFT JOIN note_original o ON o.note_id = n.id
6854
7381
  LEFT JOIN note_revised_current c ON c.note_id = n.id
@@ -6860,7 +7387,14 @@ async function exportShard(db, options) {
6860
7387
  noteQuery = `SELECT n.id, n.title, n.format, n.source, n.is_starred, n.is_archived,
6861
7388
  n.created_at, n.updated_at, n.deleted_at,
6862
7389
  o.content as original_content,
6863
- c.content as revised_content
7390
+ c.content as revised_content,
7391
+ (
7392
+ SELECT cn.collection_id
7393
+ FROM collection_note cn
7394
+ WHERE cn.note_id = n.id
7395
+ ORDER BY cn.position, cn.added_at
7396
+ LIMIT 1
7397
+ ) as collection_id
6864
7398
  FROM note n
6865
7399
  LEFT JOIN note_original o ON o.note_id = n.id
6866
7400
  LEFT JOIN note_revised_current c ON c.note_id = n.id
@@ -6892,26 +7426,28 @@ async function exportShard(db, options) {
6892
7426
  WHERE a.deleted_at IS NULL
6893
7427
  ORDER BY a.note_id, a.position, a.created_at`
6894
7428
  );
6895
- const binarySourcesByNote = /* @__PURE__ */ new Map();
7429
+ const attachmentsByNote = /* @__PURE__ */ new Map();
6896
7430
  for (const row of attachmentRows.rows) {
6897
7431
  const source = {
6898
- extracted_text: row.extracted_text ?? "",
7432
+ extracted_text: row.extracted_text,
6899
7433
  attachment: {
7434
+ // `path` is the display filename per the binary-attachment projection
7435
+ // contract — never the physical storage key (`storage_path`).
6900
7436
  id: row.id,
6901
- path: row.storage_path ?? row.filename,
7437
+ path: row.filename,
6902
7438
  mime: row.mime_type,
6903
7439
  checksum: row.content_hash,
6904
7440
  bytes: Number(row.size_bytes)
6905
7441
  }
6906
7442
  };
6907
- const sources = binarySourcesByNote.get(row.note_id) ?? [];
7443
+ const sources = attachmentsByNote.get(row.note_id) ?? [];
6908
7444
  sources.push(source);
6909
- binarySourcesByNote.set(row.note_id, sources);
7445
+ attachmentsByNote.set(row.note_id, sources);
6910
7446
  }
6911
7447
  const notes = noteRows.rows.map((row) => ({
6912
7448
  ...row,
6913
7449
  tags: tagsByNote.get(row.id) ?? [],
6914
- binary_sources: binarySourcesByNote.get(row.id)
7450
+ attachments: attachmentsByNote.get(row.id)
6915
7451
  }));
6916
7452
  const exportedNoteIds = new Set(notes.map((n) => n.id));
6917
7453
  const shardNotes = notes.map((n) => noteToShard(n));
@@ -6922,7 +7458,7 @@ async function exportShard(db, options) {
6922
7458
  for (let offset = 0; offset < shardNotes.length; offset += clusterSize) {
6923
7459
  const slice = shardNotes.slice(offset, offset + clusterSize);
6924
7460
  const href = `notes/${String(offset).padStart(6, "0")}.jsonl`;
6925
- clusters.push({ href, offset, count: slice.length });
7461
+ clusters.push({ href, offset });
6926
7462
  files.set(href, encoder.encode(slice.map((n) => JSON.stringify(n)).join("\n")));
6927
7463
  }
6928
7464
  layout = { clusters: { notes: clusters } };
@@ -6963,14 +7499,27 @@ async function exportShard(db, options) {
6963
7499
  files.set("tags.json", encoder.encode(JSON.stringify(shardTags)));
6964
7500
  components.push("tags");
6965
7501
  counts.tags = shardTags.length;
7502
+ const templateRows = await db.query(`SELECT * FROM template ORDER BY created_at, id`);
7503
+ if (templateRows.rows.length > 0) {
7504
+ const shardTemplates = templateRows.rows.map((template) => templateToShard(template));
7505
+ files.set("templates.json", encoder.encode(JSON.stringify(shardTemplates)));
7506
+ components.push("templates");
7507
+ counts.templates = shardTemplates.length;
7508
+ }
6966
7509
  const linkRows = await db.query(
6967
7510
  `SELECT * FROM link WHERE deleted_at IS NULL ORDER BY created_at`
6968
7511
  );
6969
7512
  const filteredLinks = options?.tag || options?.collectionId ? linkRows.rows.filter((l) => exportedNoteIds.has(l.source_note_id) && exportedNoteIds.has(l.target_note_id)) : linkRows.rows;
6970
- const linksJsonl = filteredLinks.map((l) => JSON.stringify(linkToShard(l))).join("\n");
7513
+ const urlLinkRows = await db.query(`SELECT * FROM link_url_target WHERE deleted_at IS NULL ORDER BY created_at`);
7514
+ const filteredUrlLinks = options?.tag || options?.collectionId ? urlLinkRows.rows.filter((l) => exportedNoteIds.has(l.source_note_id)) : urlLinkRows.rows;
7515
+ const shardLinks = [
7516
+ ...filteredLinks.map((l) => linkToShard(l)),
7517
+ ...filteredUrlLinks.map((l) => urlLinkToShard(l))
7518
+ ];
7519
+ const linksJsonl = shardLinks.map((l) => JSON.stringify(l)).join("\n");
6971
7520
  files.set("links.jsonl", encoder.encode(linksJsonl));
6972
7521
  components.push("links");
6973
- counts.links = filteredLinks.length;
7522
+ counts.links = shardLinks.length;
6974
7523
  const allNoteSkosRows = await db.query(`SELECT * FROM note_skos_tag ORDER BY created_at`);
6975
7524
  const filteredNoteSkosRows = isFiltered ? allNoteSkosRows.rows.filter((row) => exportedNoteIds.has(row.note_id)) : allNoteSkosRows.rows;
6976
7525
  const referencedConceptIds = new Set(filteredNoteSkosRows.map((row) => row.concept_id));
@@ -7011,9 +7560,27 @@ async function exportShard(db, options) {
7011
7560
  const setScoped = embeddingSetIds.length > 0;
7012
7561
  const includeMaterializedSelectors = options.includeMaterializedSelectors === true;
7013
7562
  const embSetRows = await db.query(
7014
- `SELECT * FROM embedding_set
7015
- ${setScoped ? "WHERE id = ANY($1)" : ""}
7016
- ORDER BY created_at`,
7563
+ `SELECT
7564
+ es.id, es.name, es.slug, es.description, es.purpose,
7565
+ COALESCE(es.document_count, member_counts.document_count, 0)::int AS document_count,
7566
+ COALESCE(es.embedding_count, embedding_counts.embedding_count, 0)::int AS embedding_count,
7567
+ es.is_system, es.keywords_json,
7568
+ es.model_name, es.dimensions, es.kind, es.mode, es.truncate_dimension,
7569
+ es.criteria_json, es.source_json, es.compatibility_json, es.materialization_json,
7570
+ es.freshness_json, es.created_at, es.updated_at
7571
+ FROM embedding_set es
7572
+ LEFT JOIN (
7573
+ SELECT embedding_set_id, COUNT(*)::int AS document_count
7574
+ FROM embedding_set_member
7575
+ GROUP BY embedding_set_id
7576
+ ) member_counts ON member_counts.embedding_set_id = es.id
7577
+ LEFT JOIN (
7578
+ SELECT embedding_set_id, COUNT(*)::int AS embedding_count
7579
+ FROM embedding
7580
+ GROUP BY embedding_set_id
7581
+ ) embedding_counts ON embedding_counts.embedding_set_id = es.id
7582
+ ${setScoped ? "WHERE es.id = ANY($1)" : ""}
7583
+ ORDER BY es.created_at`,
7017
7584
  setScoped ? [embeddingSetIds] : []
7018
7585
  );
7019
7586
  const exportedSetIds = new Set(embSetRows.rows.map((row) => row.id));
@@ -7028,6 +7595,17 @@ async function exportShard(db, options) {
7028
7595
  files.set("embedding_sets.json", encoder.encode(JSON.stringify(shardEmbSets)));
7029
7596
  components.push("embedding_sets");
7030
7597
  counts.embedding_sets = shardEmbSets.length;
7598
+ const embeddingConfigRows = await db.query(
7599
+ `SELECT id, name, description, model, dimension, chunk_size, chunk_overlap, is_default
7600
+ FROM embedding_config
7601
+ ORDER BY name, id`
7602
+ );
7603
+ if (embeddingConfigRows.rows.length > 0) {
7604
+ const shardEmbeddingConfigs = embeddingConfigRows.rows.map((row) => embeddingConfigToShard(row));
7605
+ files.set("embedding_configs.json", encoder.encode(JSON.stringify(shardEmbeddingConfigs)));
7606
+ components.push("embedding_configs");
7607
+ counts.embedding_configs = shardEmbeddingConfigs.length;
7608
+ }
7031
7609
  const embMemberRows = await db.query(
7032
7610
  `SELECT * FROM embedding_set_member
7033
7611
  ${setScoped ? "WHERE embedding_set_id = ANY($1)" : ""}`,
@@ -7041,12 +7619,24 @@ async function exportShard(db, options) {
7041
7619
  components.push("embedding_set_members");
7042
7620
  counts.embedding_set_members = scopedEmbMemberRows.length;
7043
7621
  const embRows = await db.query(
7044
- `SELECT * FROM embedding
7045
- ${setScoped ? "WHERE embedding_set_id = ANY($1)" : ""}
7046
- ORDER BY created_at`,
7622
+ `SELECT
7623
+ e.id, e.note_id, e.embedding_set_id, e.chunk_index,
7624
+ COALESCE(NULLIF(e.text, ''), nrc.content, no.content, '') AS text,
7625
+ e.vector,
7626
+ COALESCE(e.model, es.model_name) AS model,
7627
+ es.model_name,
7628
+ e.created_at
7629
+ FROM embedding e
7630
+ JOIN embedding_set es ON es.id = e.embedding_set_id
7631
+ LEFT JOIN note_revised_current nrc ON nrc.note_id = e.note_id
7632
+ LEFT JOIN note_original no ON no.note_id = e.note_id
7633
+ ${setScoped ? "WHERE e.embedding_set_id = ANY($1)" : ""}
7634
+ ORDER BY e.created_at`,
7047
7635
  setScoped ? [embeddingSetIds] : []
7048
7636
  );
7049
- const memberEmbeddingIds = new Set(scopedEmbMemberRows.map((member) => member.embedding_id));
7637
+ const memberEmbeddingIds = new Set(
7638
+ scopedEmbMemberRows.map((member) => member.embedding_id).filter((id) => Boolean(id))
7639
+ );
7050
7640
  const scopedEmbRows = embRows.rows.filter(
7051
7641
  (embedding) => exportedSetIds.has(embedding.embedding_set_id) && exportedNoteIds.has(embedding.note_id) && (memberEmbeddingIds.size === 0 || memberEmbeddingIds.has(embedding.id))
7052
7642
  );
@@ -7162,14 +7752,37 @@ async function exportShard(db, options) {
7162
7752
  counts,
7163
7753
  checksums,
7164
7754
  min_reader_version: "1.0.0",
7755
+ migrated_from: null,
7756
+ migration_history: [],
7165
7757
  ...layout ? { layout } : {}
7166
7758
  };
7167
7759
  files.set("manifest.json", encoder.encode(JSON.stringify(manifest, null, 2)));
7760
+ if (options?.includeBlobs && options.blobStore) {
7761
+ const packed = /* @__PURE__ */ new Set();
7762
+ for (const row of attachmentRows.rows) {
7763
+ const checksum = row.content_hash;
7764
+ if (packed.has(checksum)) continue;
7765
+ packed.add(checksum);
7766
+ const bytes = await options.blobStore.read(checksum);
7767
+ if (bytes) files.set(sidecarEntryName(checksum), bytes);
7768
+ }
7769
+ }
7168
7770
  return packTarGz(files);
7169
7771
  }
7170
7772
 
7171
- // src/shard/shard-import.ts
7773
+ // src/shard/parse.ts
7172
7774
  var decoder = new TextDecoder();
7775
+ function parseJsonlBytes(data) {
7776
+ if (!data || data.byteLength === 0) return [];
7777
+ return decoder.decode(data).split("\n").filter((line) => line.trim()).map((line) => JSON.parse(line));
7778
+ }
7779
+ function parseJsonArrayBytes(data) {
7780
+ if (!data || data.byteLength === 0) return [];
7781
+ return JSON.parse(decoder.decode(data));
7782
+ }
7783
+
7784
+ // src/shard/shard-import.ts
7785
+ var decoder2 = new TextDecoder();
7173
7786
  var DEFAULT_BATCH_SIZE = 250;
7174
7787
  async function yieldToEventLoop2() {
7175
7788
  const scheduler = globalThis.scheduler;
@@ -7191,14 +7804,17 @@ async function importShard(db, data, options) {
7191
7804
  const report = options?.onProgress;
7192
7805
  const warnings = [];
7193
7806
  const errors = [];
7194
- let droppedAttachmentCount = 0;
7195
- let notesWithDroppedAttachments = 0;
7807
+ let importedAttachmentReferenceCount = 0;
7808
+ let notesWithImportedAttachmentReferences = 0;
7809
+ const blobsToHydrate = /* @__PURE__ */ new Map();
7196
7810
  const counts = {
7197
7811
  notes: 0,
7198
7812
  collections: 0,
7813
+ templates: 0,
7199
7814
  tags: 0,
7200
7815
  links: 0,
7201
7816
  embedding_sets: 0,
7817
+ embedding_configs: 0,
7202
7818
  embedding_set_members: 0,
7203
7819
  embeddings: 0,
7204
7820
  skos_schemes: 0,
@@ -7242,7 +7858,10 @@ async function importShard(db, data, options) {
7242
7858
  }
7243
7859
  let manifest;
7244
7860
  try {
7245
- manifest = JSON.parse(decoder.decode(manifestData));
7861
+ manifest = JSON.parse(decoder2.decode(manifestData));
7862
+ if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) {
7863
+ throw new Error("manifest must be a JSON object");
7864
+ }
7246
7865
  } catch {
7247
7866
  return {
7248
7867
  success: false,
@@ -7253,7 +7872,7 @@ async function importShard(db, data, options) {
7253
7872
  duration_ms: performance.now() - start
7254
7873
  };
7255
7874
  }
7256
- if (manifest.min_reader_version && manifest.min_reader_version > CURRENT_SHARD_VERSION) {
7875
+ if (manifest.min_reader_version && compareShardVersions(manifest.min_reader_version, CURRENT_SHARD_VERSION) > 0) {
7257
7876
  return {
7258
7877
  success: false,
7259
7878
  counts,
@@ -7266,6 +7885,16 @@ async function importShard(db, data, options) {
7266
7885
  };
7267
7886
  }
7268
7887
  report?.({ phase: "validate", done: 0, total: 1 });
7888
+ if (!manifest.checksums || typeof manifest.checksums !== "object") {
7889
+ return {
7890
+ success: false,
7891
+ counts,
7892
+ skipped,
7893
+ warnings,
7894
+ errors: ["Invalid manifest.json: checksums must be an object"],
7895
+ duration_ms: performance.now() - start
7896
+ };
7897
+ }
7269
7898
  const checksumResult = await validateChecksums(manifest.checksums, files);
7270
7899
  if (!checksumResult.valid) {
7271
7900
  return {
@@ -7278,25 +7907,61 @@ async function importShard(db, data, options) {
7278
7907
  };
7279
7908
  }
7280
7909
  report?.({ phase: "validate", done: 1, total: 1 });
7910
+ const sidecarBlobs = options?.blobStore ? collectSidecarBlobs(files) : null;
7281
7911
  const noteClusters = manifest.layout?.clusters?.notes;
7282
- const parsedNotes = noteClusters && noteClusters.length > 0 ? [...noteClusters].sort((a, b) => a.offset - b.offset).flatMap((ref) => parseJsonl(files.get(ref.href))) : parseJsonl(files.get("notes.jsonl"));
7283
- const parsedCollections = parseJsonArray(files.get("collections.json"));
7284
- parseJsonArray(files.get("tags.json"));
7285
- const parsedLinks = parseJsonl(files.get("links.jsonl"));
7286
- const parsedEmbSets = parseJsonArray(files.get("embedding_sets.json"));
7287
- const parsedEmbMembers = parseJsonl(
7288
- files.get("embedding_set_members.jsonl")
7289
- );
7290
- const parsedEmbeddings = parseJsonl(files.get("embeddings.jsonl"));
7291
- const parsedSkosSchemes = parseJsonArray(files.get("skos_schemes.json"));
7292
- const parsedSkosConcepts = parseJsonArray(files.get("skos_concepts.json"));
7293
- const parsedSkosRelations = parseJsonl(files.get("skos_relations.jsonl"));
7294
- const parsedNoteSkosTags = parseJsonl(files.get("note_skos_tags.jsonl"));
7295
- const parsedProvenanceEdges = parseJsonl(files.get("provenance_edges.jsonl"));
7296
- const parsedGraphSources = parseJsonArray(files.get("graph_sources.json"));
7297
- const parsedGraphEdges = parseJsonl(files.get("graph_edges.jsonl"));
7298
- const parsedCommunitySets = parseJsonArray(files.get("communities.json"));
7299
- const parsedCommunityAssignments = parseJsonl(files.get("community_assignments.jsonl"));
7912
+ const parseComponent = (name, parse) => {
7913
+ try {
7914
+ return parse();
7915
+ } catch (err) {
7916
+ throw new Error(`${name}: ${err instanceof Error ? err.message : String(err)}`);
7917
+ }
7918
+ };
7919
+ let parsedNotes;
7920
+ let parsedCollections;
7921
+ let parsedTemplates;
7922
+ let parsedLinks;
7923
+ let parsedEmbSets;
7924
+ let parsedEmbConfigs;
7925
+ let parsedEmbMembers;
7926
+ let parsedEmbeddings;
7927
+ let parsedSkosSchemes;
7928
+ let parsedSkosConcepts;
7929
+ let parsedSkosRelations;
7930
+ let parsedNoteSkosTags;
7931
+ let parsedProvenanceEdges;
7932
+ let parsedGraphSources;
7933
+ let parsedGraphEdges;
7934
+ let parsedCommunitySets;
7935
+ let parsedCommunityAssignments;
7936
+ try {
7937
+ parsedNotes = parseComponent("notes", () => noteClusters && noteClusters.length > 0 ? [...noteClusters].sort((a, b) => a.offset - b.offset).flatMap((ref) => parseJsonlBytes(files.get(ref.href))) : parseJsonlBytes(files.get("notes.jsonl")));
7938
+ parsedCollections = parseComponent("collections.json", () => parseJsonArrayBytes(files.get("collections.json")));
7939
+ parseComponent("tags.json", () => parseJsonArrayBytes(files.get("tags.json")));
7940
+ parsedTemplates = parseComponent("templates.json", () => parseJsonArrayBytes(files.get("templates.json")));
7941
+ parsedLinks = parseComponent("links.jsonl", () => parseJsonlBytes(files.get("links.jsonl")));
7942
+ parsedEmbSets = parseComponent("embedding_sets.json", () => parseJsonArrayBytes(files.get("embedding_sets.json")));
7943
+ parsedEmbConfigs = parseComponent("embedding_configs.json", () => parseJsonArrayBytes(files.get("embedding_configs.json")));
7944
+ parsedEmbMembers = parseComponent("embedding_set_members.jsonl", () => parseJsonlBytes(files.get("embedding_set_members.jsonl")));
7945
+ parsedEmbeddings = parseComponent("embeddings.jsonl", () => parseJsonlBytes(files.get("embeddings.jsonl")));
7946
+ parsedSkosSchemes = parseComponent("skos_schemes.json", () => parseJsonArrayBytes(files.get("skos_schemes.json")));
7947
+ parsedSkosConcepts = parseComponent("skos_concepts.json", () => parseJsonArrayBytes(files.get("skos_concepts.json")));
7948
+ parsedSkosRelations = parseComponent("skos_relations.jsonl", () => parseJsonlBytes(files.get("skos_relations.jsonl")));
7949
+ parsedNoteSkosTags = parseComponent("note_skos_tags.jsonl", () => parseJsonlBytes(files.get("note_skos_tags.jsonl")));
7950
+ parsedProvenanceEdges = parseComponent("provenance_edges.jsonl", () => parseJsonlBytes(files.get("provenance_edges.jsonl")));
7951
+ parsedGraphSources = parseComponent("graph_sources.json", () => parseJsonArrayBytes(files.get("graph_sources.json")));
7952
+ parsedGraphEdges = parseComponent("graph_edges.jsonl", () => parseJsonlBytes(files.get("graph_edges.jsonl")));
7953
+ parsedCommunitySets = parseComponent("communities.json", () => parseJsonArrayBytes(files.get("communities.json")));
7954
+ parsedCommunityAssignments = parseComponent("community_assignments.jsonl", () => parseJsonlBytes(files.get("community_assignments.jsonl")));
7955
+ } catch (err) {
7956
+ return {
7957
+ success: false,
7958
+ counts,
7959
+ skipped,
7960
+ warnings,
7961
+ errors: [`Failed to parse shard component: ${err instanceof Error ? err.message : String(err)}`],
7962
+ duration_ms: performance.now() - start
7963
+ };
7964
+ }
7300
7965
  const knownFiles = /* @__PURE__ */ new Set([
7301
7966
  "manifest.json",
7302
7967
  "notes.jsonl",
@@ -7305,7 +7970,6 @@ async function importShard(db, data, options) {
7305
7970
  "links.jsonl",
7306
7971
  "embedding_sets.json",
7307
7972
  "embedding_set_members.jsonl",
7308
- "embedding_configs.json",
7309
7973
  "embeddings.jsonl",
7310
7974
  "templates.json",
7311
7975
  "skos_schemes.json",
@@ -7323,15 +7987,36 @@ async function importShard(db, data, options) {
7323
7987
  warnings.push(`Unknown component skipped: ${filename}`);
7324
7988
  }
7325
7989
  }
7326
- if (files.has("templates.json")) {
7327
- warnings.push("templates.json skipped (not supported in browser)");
7328
- }
7329
7990
  const conflictClause = strategy === "skip" ? "ON CONFLICT DO NOTHING" : "";
7330
7991
  try {
7331
7992
  await db.transaction(async (tx) => {
7993
+ const preexistingEmbeddingMembers = /* @__PURE__ */ new Set();
7994
+ if (strategy === "skip") {
7995
+ for (const member of parsedEmbMembers) {
7996
+ const existing = await tx.query(
7997
+ "SELECT 1 FROM embedding_set_member WHERE embedding_set_id = $1 AND note_id = $2",
7998
+ [member.embedding_set_id, member.note_id]
7999
+ );
8000
+ if (existing.rows.length > 0) {
8001
+ preexistingEmbeddingMembers.add(`${member.embedding_set_id}\0${member.note_id}`);
8002
+ }
8003
+ }
8004
+ }
8005
+ const skipExisting = async (key, sql, params) => {
8006
+ if (strategy !== "skip") return false;
8007
+ const existing = await tx.query(sql, params);
8008
+ if (existing.rows.length === 0) return false;
8009
+ skipped[key] = (skipped[key] ?? 0) + 1;
8010
+ return true;
8011
+ };
7332
8012
  report?.({ phase: "collections", done: 0, total: parsedCollections.length });
7333
8013
  for (const [index, shardCol] of parsedCollections.entries()) {
7334
8014
  const col = collectionFromShard(shardCol);
8015
+ if (await skipExisting("collections", "SELECT 1 FROM collection WHERE id = $1", [col.id])) {
8016
+ report?.({ phase: "collections", done: index + 1, total: parsedCollections.length });
8017
+ await maybeYield2(index + 1, batchSize);
8018
+ continue;
8019
+ }
7335
8020
  if (strategy === "replace") {
7336
8021
  await tx.query(
7337
8022
  `INSERT INTO collection (id, name, description, parent_id, created_at)
@@ -7350,9 +8035,38 @@ async function importShard(db, data, options) {
7350
8035
  report?.({ phase: "collections", done: index + 1, total: parsedCollections.length });
7351
8036
  await maybeYield2(index + 1, batchSize);
7352
8037
  }
8038
+ report?.({ phase: "templates", done: 0, total: parsedTemplates.length });
8039
+ for (const [index, template] of parsedTemplates.entries()) {
8040
+ await tx.query(
8041
+ `INSERT INTO template (
8042
+ id, name, description, content, format, default_tags, collection_id, created_at, updated_at
8043
+ )
8044
+ VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9)
8045
+ ${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET name = $2, description = $3, content = $4, format = $5, default_tags = $6::jsonb, collection_id = $7, created_at = $8, updated_at = $9" : conflictClause}`,
8046
+ [
8047
+ template.id,
8048
+ template.name,
8049
+ template.description,
8050
+ template.content,
8051
+ template.format,
8052
+ JSON.stringify(template.default_tags),
8053
+ template.collection_id,
8054
+ template.created_at,
8055
+ template.updated_at
8056
+ ]
8057
+ );
8058
+ counts.templates++;
8059
+ report?.({ phase: "templates", done: index + 1, total: parsedTemplates.length });
8060
+ await maybeYield2(index + 1, batchSize);
8061
+ }
7353
8062
  report?.({ phase: "notes", done: 0, total: parsedNotes.length });
7354
8063
  for (const [index, shardNote] of parsedNotes.entries()) {
7355
8064
  const note = noteFromShard(shardNote);
8065
+ if (await skipExisting("notes", "SELECT 1 FROM note WHERE id = $1", [note.id])) {
8066
+ report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
8067
+ await maybeYield2(index + 1, batchSize);
8068
+ continue;
8069
+ }
7356
8070
  const contentHash = computeHash(new TextEncoder().encode(note.original_content));
7357
8071
  if (strategy === "replace") {
7358
8072
  await tx.query(
@@ -7427,23 +8141,98 @@ async function importShard(db, data, options) {
7427
8141
  [generateId(), note.id, tag]
7428
8142
  );
7429
8143
  }
7430
- if (note.binary_sources?.length) {
7431
- droppedAttachmentCount += note.binary_sources.length;
7432
- notesWithDroppedAttachments++;
8144
+ if (note.collection_id) {
8145
+ await tx.query(
8146
+ `INSERT INTO collection_note (collection_id, note_id)
8147
+ VALUES ($1, $2)
8148
+ ON CONFLICT (collection_id, note_id) DO NOTHING`,
8149
+ [note.collection_id, note.id]
8150
+ );
8151
+ }
8152
+ if (note.attachments?.length) {
8153
+ notesWithImportedAttachmentReferences++;
8154
+ for (let position = 0; position < note.attachments.length; position += 1) {
8155
+ const projection = note.attachments[position];
8156
+ const ref = projection.attachment;
8157
+ const existingBlob = await tx.query(
8158
+ `SELECT id FROM attachment_blob WHERE content_hash = $1`,
8159
+ [ref.checksum]
8160
+ );
8161
+ const blobId = existingBlob.rows[0]?.id ?? generateId();
8162
+ if (!existingBlob.rows.length) {
8163
+ await tx.query(
8164
+ `INSERT INTO attachment_blob (id, content_hash, size_bytes, storage_path)
8165
+ VALUES ($1, $2, $3, $4)
8166
+ ON CONFLICT (content_hash) DO NOTHING`,
8167
+ // storage_path stays NULL: the browser edition addresses blobs
8168
+ // by content_hash via the BlobStore, not a filesystem key, and
8169
+ // `ref.path` is the display filename (never a storage locator).
8170
+ [blobId, ref.checksum, ref.bytes, null]
8171
+ );
8172
+ }
8173
+ const filename = ref.path.split("/").filter(Boolean).pop() ?? ref.path;
8174
+ if (strategy === "replace") {
8175
+ await tx.query(
8176
+ `INSERT INTO attachment (id, note_id, blob_id, filename, mime_type, extracted_text, position)
8177
+ VALUES ($1, $2, $3, $4, $5, $6, $7)
8178
+ ON CONFLICT (id) DO UPDATE SET
8179
+ note_id = $2,
8180
+ blob_id = $3,
8181
+ filename = $4,
8182
+ mime_type = $5,
8183
+ extracted_text = $6,
8184
+ position = $7,
8185
+ deleted_at = NULL`,
8186
+ [ref.id, note.id, blobId, filename, ref.mime, projection.extracted_text, position]
8187
+ );
8188
+ } else {
8189
+ await tx.query(
8190
+ `INSERT INTO attachment (id, note_id, blob_id, filename, mime_type, extracted_text, position)
8191
+ VALUES ($1, $2, $3, $4, $5, $6, $7) ${conflictClause}`,
8192
+ [ref.id, note.id, blobId, filename, ref.mime, projection.extracted_text, position]
8193
+ );
8194
+ }
8195
+ let hydrated = false;
8196
+ if (sidecarBlobs) {
8197
+ if (blobsToHydrate.has(ref.checksum)) {
8198
+ hydrated = true;
8199
+ } else {
8200
+ const bytes = sidecarBlobs.get(blobChecksumToHex(ref.checksum));
8201
+ if (bytes) {
8202
+ if (computeBlobHash(bytes) === ref.checksum) {
8203
+ blobsToHydrate.set(ref.checksum, bytes);
8204
+ hydrated = true;
8205
+ } else {
8206
+ warnings.push(
8207
+ `Sidecar blob for attachment ${ref.id} failed BLAKE3 integrity check (expected ${ref.checksum}); imported as reference-only.`
8208
+ );
8209
+ }
8210
+ }
8211
+ }
8212
+ }
8213
+ if (!hydrated) {
8214
+ importedAttachmentReferenceCount++;
8215
+ }
8216
+ }
7433
8217
  }
7434
8218
  counts.notes++;
7435
8219
  report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
7436
8220
  await maybeYield2(index + 1, batchSize);
7437
8221
  }
7438
- if (droppedAttachmentCount > 0) {
8222
+ if (importedAttachmentReferenceCount > 0) {
7439
8223
  warnings.push(
7440
- `${droppedAttachmentCount} attachment(s) across ${notesWithDroppedAttachments} note(s) were not imported: shards currently carry attachment references (metadata + checksum) but not the binary content, so attachment bytes cannot be restored. Tracking: #237 (attachment round-trip) / server #1013 (binary contract).`
8224
+ `${importedAttachmentReferenceCount} attachment reference(s) across ${notesWithImportedAttachmentReferences} note(s) were imported as metadata only: these shard records carry extracted text plus attachment metadata but no matching byte-sidecar entry, so their BlobStore bytes are unavailable (getBlob() returns null). Export a self-contained shard (\`includeBlobs\` + a \`blobStore\`) and import with a \`blobStore\` to hydrate bytes. Tracking: #271 / server #1046.`
7441
8225
  );
7442
8226
  }
7443
8227
  const totalSkos = parsedSkosSchemes.length + parsedSkosConcepts.length + parsedSkosRelations.length + parsedNoteSkosTags.length;
7444
8228
  let doneSkos = 0;
7445
8229
  report?.({ phase: "skos", done: doneSkos, total: totalSkos });
7446
8230
  for (const scheme of parsedSkosSchemes) {
8231
+ if (await skipExisting("skos_schemes", "SELECT 1 FROM skos_scheme WHERE id = $1", [scheme.id])) {
8232
+ report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
8233
+ await maybeYield2(doneSkos, batchSize);
8234
+ continue;
8235
+ }
7447
8236
  if (strategy === "replace") {
7448
8237
  await tx.query(
7449
8238
  `INSERT INTO skos_scheme (id, title, description, created_at, updated_at)
@@ -7463,6 +8252,11 @@ async function importShard(db, data, options) {
7463
8252
  await maybeYield2(doneSkos, batchSize);
7464
8253
  }
7465
8254
  for (const concept of parsedSkosConcepts) {
8255
+ if (await skipExisting("skos_concepts", "SELECT 1 FROM skos_concept WHERE id = $1", [concept.id])) {
8256
+ report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
8257
+ await maybeYield2(doneSkos, batchSize);
8258
+ continue;
8259
+ }
7466
8260
  const altLabels = JSON.stringify(concept.alt_labels ?? []);
7467
8261
  if (strategy === "replace") {
7468
8262
  await tx.query(
@@ -7485,7 +8279,34 @@ async function importShard(db, data, options) {
7485
8279
  report?.({ phase: "links", done: 0, total: parsedLinks.length });
7486
8280
  for (const [index, shardLink] of parsedLinks.entries()) {
7487
8281
  const link = linkFromShard(shardLink);
7488
- if (strategy === "replace") {
8282
+ if (!link.target_note_id) {
8283
+ if (!link.to_url) {
8284
+ skipped.links = (skipped.links ?? 0) + 1;
8285
+ warnings.push(`Shard link skipped: ${link.id} has neither to_note_id nor to_url.`);
8286
+ report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
8287
+ await maybeYield2(index + 1, batchSize);
8288
+ continue;
8289
+ }
8290
+ const metadata = link.metadata == null ? null : JSON.stringify(link.metadata);
8291
+ await tx.query(
8292
+ `INSERT INTO link_url_target (
8293
+ id, source_note_id, to_url, link_type, confidence, metadata_json, created_at
8294
+ )
8295
+ VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7)
8296
+ ${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET source_note_id = $2, to_url = $3, link_type = $4, confidence = $5, metadata_json = $6::jsonb, created_at = $7" : conflictClause}`,
8297
+ [link.id, link.source_note_id, link.to_url, link.link_type, link.confidence, metadata, link.created_at]
8298
+ );
8299
+ counts.links++;
8300
+ report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
8301
+ await maybeYield2(index + 1, batchSize);
8302
+ continue;
8303
+ }
8304
+ if (await skipExisting("links", "SELECT 1 FROM link WHERE id = $1", [link.id])) {
8305
+ report?.({ phase: "links", done: index + 1, total: parsedLinks.length });
8306
+ await maybeYield2(index + 1, batchSize);
8307
+ continue;
8308
+ }
8309
+ if (strategy === "replace") {
7489
8310
  await tx.query(
7490
8311
  `INSERT INTO link (id, source_note_id, target_note_id, link_type, confidence, created_at)
7491
8312
  VALUES ($1, $2, $3, $4, $5, $6)
@@ -7504,6 +8325,11 @@ async function importShard(db, data, options) {
7504
8325
  await maybeYield2(index + 1, batchSize);
7505
8326
  }
7506
8327
  for (const relation of parsedSkosRelations) {
8328
+ if (await skipExisting("skos_relations", "SELECT 1 FROM skos_concept_relation WHERE id = $1", [relation.id])) {
8329
+ report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
8330
+ await maybeYield2(doneSkos, batchSize);
8331
+ continue;
8332
+ }
7507
8333
  if (strategy === "replace") {
7508
8334
  await tx.query(
7509
8335
  `INSERT INTO skos_concept_relation (id, source_concept_id, target_concept_id, relation_type, created_at)
@@ -7523,6 +8349,11 @@ async function importShard(db, data, options) {
7523
8349
  await maybeYield2(doneSkos, batchSize);
7524
8350
  }
7525
8351
  for (const tag of parsedNoteSkosTags) {
8352
+ if (await skipExisting("note_skos_tags", "SELECT 1 FROM note_skos_tag WHERE note_id = $1 AND concept_id = $2", [tag.note_id, tag.concept_id])) {
8353
+ report?.({ phase: "skos", done: ++doneSkos, total: totalSkos });
8354
+ await maybeYield2(doneSkos, batchSize);
8355
+ continue;
8356
+ }
7526
8357
  await tx.query(
7527
8358
  `INSERT INTO note_skos_tag (id, note_id, concept_id, created_at)
7528
8359
  VALUES ($1, $2, $3, $4)
@@ -7535,6 +8366,11 @@ async function importShard(db, data, options) {
7535
8366
  }
7536
8367
  report?.({ phase: "provenance", done: 0, total: parsedProvenanceEdges.length });
7537
8368
  for (const [index, edge] of parsedProvenanceEdges.entries()) {
8369
+ if (await skipExisting("provenance_edges", "SELECT 1 FROM provenance_edge WHERE id = $1", [edge.id])) {
8370
+ report?.({ phase: "provenance", done: index + 1, total: parsedProvenanceEdges.length });
8371
+ await maybeYield2(index + 1, batchSize);
8372
+ continue;
8373
+ }
7538
8374
  const attributes = edge.attributes === null ? null : JSON.stringify(edge.attributes);
7539
8375
  if (strategy === "replace") {
7540
8376
  await tx.query(
@@ -7554,27 +8390,59 @@ async function importShard(db, data, options) {
7554
8390
  report?.({ phase: "provenance", done: index + 1, total: parsedProvenanceEdges.length });
7555
8391
  await maybeYield2(index + 1, batchSize);
7556
8392
  }
8393
+ report?.({ phase: "embedding_configs", done: 0, total: parsedEmbConfigs.length });
8394
+ for (const [index, config] of parsedEmbConfigs.entries()) {
8395
+ await tx.query(
8396
+ `INSERT INTO embedding_config (
8397
+ id, name, description, model, dimension, chunk_size, chunk_overlap, is_default
8398
+ )
8399
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
8400
+ ${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET name = $2, description = $3, model = $4, dimension = $5, chunk_size = $6, chunk_overlap = $7, is_default = $8" : conflictClause}`,
8401
+ [
8402
+ config.id,
8403
+ config.name,
8404
+ config.description,
8405
+ config.model,
8406
+ config.dimension,
8407
+ config.chunk_size,
8408
+ config.chunk_overlap,
8409
+ config.is_default
8410
+ ]
8411
+ );
8412
+ counts.embedding_configs++;
8413
+ report?.({ phase: "embedding_configs", done: index + 1, total: parsedEmbConfigs.length });
8414
+ await maybeYield2(index + 1, batchSize);
8415
+ }
7557
8416
  report?.({ phase: "embedding_sets", done: 0, total: parsedEmbSets.length });
7558
8417
  for (const [index, shardSet] of parsedEmbSets.entries()) {
7559
- const set = embeddingSetFromShard(shardSet);
8418
+ const set = embeddingSetFromShard(shardSet, manifest.created_at);
8419
+ if (await skipExisting("embedding_sets", "SELECT 1 FROM embedding_set WHERE id = $1", [set.id])) {
8420
+ report?.({ phase: "embedding_sets", done: index + 1, total: parsedEmbSets.length });
8421
+ await maybeYield2(index + 1, batchSize);
8422
+ continue;
8423
+ }
7560
8424
  if (strategy === "replace") {
7561
8425
  await tx.query(
7562
8426
  `INSERT INTO embedding_set (
7563
- id, name, purpose, model_name, dimensions, kind, mode, truncate_dimension,
8427
+ id, name, slug, description, purpose, document_count, embedding_count, is_system, keywords_json,
8428
+ model_name, dimensions, kind, mode, truncate_dimension,
7564
8429
  criteria_json, source_json, compatibility_json, materialization_json, freshness_json, created_at, updated_at
7565
- ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb, $11::jsonb, $12::jsonb, $13::jsonb, $14, COALESCE($15::timestamptz, $14::timestamptz))
7566
- ON CONFLICT (id) DO UPDATE SET name = $2, purpose = $3, model_name = $4, dimensions = $5,
7567
- kind = $6, mode = $7, truncate_dimension = $8, criteria_json = $9::jsonb, source_json = $10::jsonb,
7568
- compatibility_json = $11::jsonb, materialization_json = $12::jsonb, freshness_json = $13::jsonb, updated_at = COALESCE($15::timestamptz, $14::timestamptz)`,
7569
- [set.id, set.name, set.purpose, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
8430
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11, $12, $13, $14, $15::jsonb, $16::jsonb, $17::jsonb, $18::jsonb, $19::jsonb, $20, COALESCE($21::timestamptz, $20::timestamptz))
8431
+ ON CONFLICT (id) DO UPDATE SET name = $2, slug = $3, description = $4, purpose = $5,
8432
+ document_count = $6, embedding_count = $7, is_system = $8, keywords_json = $9::jsonb,
8433
+ model_name = $10, dimensions = $11, kind = $12, mode = $13, truncate_dimension = $14,
8434
+ criteria_json = $15::jsonb, source_json = $16::jsonb, compatibility_json = $17::jsonb,
8435
+ materialization_json = $18::jsonb, freshness_json = $19::jsonb, updated_at = COALESCE($21::timestamptz, $20::timestamptz)`,
8436
+ [set.id, set.name, set.slug, set.description, set.purpose, set.document_count, set.embedding_count, set.is_system, set.keywords_json, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
7570
8437
  );
7571
8438
  } else {
7572
8439
  await tx.query(
7573
8440
  `INSERT INTO embedding_set (
7574
- id, name, purpose, model_name, dimensions, kind, mode, truncate_dimension,
8441
+ id, name, slug, description, purpose, document_count, embedding_count, is_system, keywords_json,
8442
+ model_name, dimensions, kind, mode, truncate_dimension,
7575
8443
  criteria_json, source_json, compatibility_json, materialization_json, freshness_json, created_at, updated_at
7576
- ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10::jsonb, $11::jsonb, $12::jsonb, $13::jsonb, $14, COALESCE($15::timestamptz, $14::timestamptz)) ${conflictClause}`,
7577
- [set.id, set.name, set.purpose, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
8444
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11, $12, $13, $14, $15::jsonb, $16::jsonb, $17::jsonb, $18::jsonb, $19::jsonb, $20, COALESCE($21::timestamptz, $20::timestamptz)) ${conflictClause}`,
8445
+ [set.id, set.name, set.slug, set.description, set.purpose, set.document_count, set.embedding_count, set.is_system, set.keywords_json, set.model_name, set.dimensions, set.kind, set.mode, set.truncate_dimension, set.criteria_json, set.source_json, set.compatibility_json, set.materialization_json, set.freshness_json, set.created_at, set.updated_at]
7578
8446
  );
7579
8447
  }
7580
8448
  counts.embedding_sets++;
@@ -7584,20 +8452,38 @@ async function importShard(db, data, options) {
7584
8452
  report?.({ phase: "embeddings", done: 0, total: parsedEmbeddings.length });
7585
8453
  for (const [index, shardEmb] of parsedEmbeddings.entries()) {
7586
8454
  const emb = embeddingFromShard(shardEmb);
8455
+ const embeddingSetId = emb.embedding_set_id ?? await resolveEmbeddingSetIdForServerEmbedding(tx, emb.model, emb.vector);
8456
+ if (await skipExisting("embeddings", "SELECT 1 FROM embedding WHERE id = $1", [emb.id])) {
8457
+ report?.({ phase: "embeddings", done: index + 1, total: parsedEmbeddings.length });
8458
+ await maybeYield2(index + 1, batchSize);
8459
+ continue;
8460
+ }
7587
8461
  if (strategy === "replace") {
7588
8462
  await tx.query(
7589
- `INSERT INTO embedding (id, note_id, embedding_set_id, vector, created_at)
7590
- VALUES ($1, $2, $3, $4, $5)
7591
- ON CONFLICT (id) DO UPDATE SET vector = $4`,
7592
- [emb.id, emb.note_id, emb.embedding_set_id, emb.vector, emb.created_at]
8463
+ `INSERT INTO embedding (id, note_id, embedding_set_id, chunk_index, text, vector, model, created_at)
8464
+ VALUES ($1, $2, $3, $4, $5, $6, $7, COALESCE($8::timestamptz, now()))
8465
+ ON CONFLICT (id) DO UPDATE SET embedding_set_id = $3, chunk_index = $4, text = $5, vector = $6, model = $7`,
8466
+ [emb.id, emb.note_id, embeddingSetId, emb.chunk_index, emb.text, emb.vector, emb.model, emb.created_at]
7593
8467
  );
7594
8468
  } else {
7595
8469
  await tx.query(
7596
- `INSERT INTO embedding (id, note_id, embedding_set_id, vector, created_at)
7597
- VALUES ($1, $2, $3, $4, $5) ${conflictClause}`,
7598
- [emb.id, emb.note_id, emb.embedding_set_id, emb.vector, emb.created_at]
8470
+ `INSERT INTO embedding (id, note_id, embedding_set_id, chunk_index, text, vector, model, created_at)
8471
+ VALUES ($1, $2, $3, $4, $5, $6, $7, COALESCE($8::timestamptz, now())) ${conflictClause}`,
8472
+ [emb.id, emb.note_id, embeddingSetId, emb.chunk_index, emb.text, emb.vector, emb.model, emb.created_at]
7599
8473
  );
7600
8474
  }
8475
+ await tx.query(
8476
+ `INSERT INTO embedding_set_member (
8477
+ embedding_set_id, note_id, embedding_id, membership_type, added_at, added_by
8478
+ ) VALUES ($1, $2, $3, 'materialized', COALESCE($4::timestamptz, now()), 'shard-import')
8479
+ ON CONFLICT (embedding_set_id, note_id) DO UPDATE SET
8480
+ embedding_id = EXCLUDED.embedding_id,
8481
+ membership_type = CASE
8482
+ WHEN embedding_set_member.membership_type = 'auto' THEN 'materialized'
8483
+ ELSE embedding_set_member.membership_type
8484
+ END`,
8485
+ [embeddingSetId, emb.note_id, emb.id, emb.created_at]
8486
+ );
7601
8487
  counts.embeddings++;
7602
8488
  report?.({ phase: "embeddings", done: index + 1, total: parsedEmbeddings.length });
7603
8489
  await maybeYield2(index + 1, batchSize);
@@ -7606,6 +8492,11 @@ async function importShard(db, data, options) {
7606
8492
  let doneGraph = 0;
7607
8493
  report?.({ phase: "graph", done: doneGraph, total: totalGraph });
7608
8494
  for (const source of parsedGraphSources) {
8495
+ if (await skipExisting("graph_sources", "SELECT 1 FROM graph_source WHERE id = $1", [source.id])) {
8496
+ report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
8497
+ await maybeYield2(doneGraph, batchSize);
8498
+ continue;
8499
+ }
7609
8500
  const parameters = source.parameters == null ? null : JSON.stringify(source.parameters);
7610
8501
  const freshness = JSON.stringify({ ...source.freshness ?? {}, status: "unknown" });
7611
8502
  await tx.query(
@@ -7621,6 +8512,15 @@ async function importShard(db, data, options) {
7621
8512
  await maybeYield2(doneGraph, batchSize);
7622
8513
  }
7623
8514
  for (const edge of parsedGraphEdges) {
8515
+ if (await skipExisting(
8516
+ "graph_edges",
8517
+ "SELECT 1 FROM graph_edge_artifact WHERE graph_source_id = $1 AND from_note_id = $2 AND to_note_id = $3 AND kind = $4",
8518
+ [edge.graph_source_id, edge.from_note_id, edge.to_note_id, edge.kind]
8519
+ )) {
8520
+ report?.({ phase: "graph", done: ++doneGraph, total: totalGraph });
8521
+ await maybeYield2(doneGraph, batchSize);
8522
+ continue;
8523
+ }
7624
8524
  const metadata = edge.metadata == null ? null : JSON.stringify(edge.metadata);
7625
8525
  await tx.query(
7626
8526
  `INSERT INTO graph_edge_artifact (graph_source_id, from_note_id, to_note_id, weight, kind, rank, metadata_json)
@@ -7638,16 +8538,22 @@ async function importShard(db, data, options) {
7638
8538
  for (const set of parsedCommunitySets) {
7639
8539
  const parameters = set.parameters == null ? null : JSON.stringify(set.parameters);
7640
8540
  const freshness = JSON.stringify({ ...set.freshness ?? {}, status: "unknown" });
7641
- await tx.query(
8541
+ const skippedSet = await skipExisting("community_sets", "SELECT 1 FROM community_set WHERE id = $1", [set.id]);
8542
+ if (!skippedSet) await tx.query(
7642
8543
  `INSERT INTO community_set (id, graph_source_id, name, source_type, algorithm, parameters_json, input_hash, freshness_json, created_at)
7643
8544
  VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb, $9)
7644
8545
  ${strategy === "replace" ? "ON CONFLICT (id) DO UPDATE SET graph_source_id = $2, name = $3, source_type = $4, algorithm = $5, parameters_json = $6::jsonb, input_hash = $7, freshness_json = $8::jsonb, created_at = $9" : conflictClause}`,
7645
8546
  [set.id, set.graph_source_id, set.name, set.source_type, set.algorithm ?? null, parameters, set.input_hash, freshness, set.created_at]
7646
8547
  );
7647
- counts.community_sets++;
8548
+ if (!skippedSet) counts.community_sets++;
7648
8549
  report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
7649
8550
  await maybeYield2(doneCommunities, batchSize);
7650
8551
  for (const community of set.communities ?? []) {
8552
+ if (await skipExisting("communities", "SELECT 1 FROM community WHERE community_set_id = $1 AND id = $2", [set.id, community.id])) {
8553
+ report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
8554
+ await maybeYield2(doneCommunities, batchSize);
8555
+ continue;
8556
+ }
7651
8557
  const metadata = community.metadata == null ? null : JSON.stringify(community.metadata);
7652
8558
  await tx.query(
7653
8559
  `INSERT INTO community (community_set_id, id, label, rank, size, confidence, representative_note_ids, metadata_json)
@@ -7661,6 +8567,11 @@ async function importShard(db, data, options) {
7661
8567
  }
7662
8568
  }
7663
8569
  for (const assignment of parsedCommunityAssignments) {
8570
+ if (await skipExisting("community_assignments", "SELECT 1 FROM community_assignment WHERE community_set_id = $1 AND note_id = $2", [assignment.community_set_id, assignment.note_id])) {
8571
+ report?.({ phase: "communities", done: ++doneCommunities, total: totalCommunities });
8572
+ await maybeYield2(doneCommunities, batchSize);
8573
+ continue;
8574
+ }
7664
8575
  const metadata = assignment.metadata == null ? null : JSON.stringify(assignment.metadata);
7665
8576
  await tx.query(
7666
8577
  `INSERT INTO community_assignment (community_set_id, community_id, note_id, confidence, source_type, metadata_json)
@@ -7674,10 +8585,38 @@ async function importShard(db, data, options) {
7674
8585
  }
7675
8586
  report?.({ phase: "embedding_set_members", done: 0, total: parsedEmbMembers.length });
7676
8587
  for (const [index, member] of parsedEmbMembers.entries()) {
8588
+ if (strategy === "skip" && preexistingEmbeddingMembers.has(`${member.embedding_set_id}\0${member.note_id}`)) {
8589
+ skipped.embedding_set_members = (skipped.embedding_set_members ?? 0) + 1;
8590
+ report?.({ phase: "embedding_set_members", done: index + 1, total: parsedEmbMembers.length });
8591
+ await maybeYield2(index + 1, batchSize);
8592
+ continue;
8593
+ }
8594
+ let embeddingId = member.embedding_id ?? null;
8595
+ if (!embeddingId) {
8596
+ const resolvedEmbedding = await tx.query(
8597
+ `SELECT id FROM embedding WHERE embedding_set_id = $1 AND note_id = $2 ORDER BY created_at DESC LIMIT 1`,
8598
+ [member.embedding_set_id, member.note_id]
8599
+ );
8600
+ embeddingId = resolvedEmbedding.rows[0]?.id ?? null;
8601
+ }
7677
8602
  await tx.query(
7678
- `INSERT INTO embedding_set_member (embedding_set_id, note_id, embedding_id)
7679
- VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`,
7680
- [member.embedding_set_id, member.note_id, member.embedding_id]
8603
+ `INSERT INTO embedding_set_member (
8604
+ embedding_set_id, note_id, embedding_id, membership_type, added_at, added_by
8605
+ )
8606
+ VALUES ($1, $2, $3, $4, $5, $6)
8607
+ ON CONFLICT (embedding_set_id, note_id) DO UPDATE SET
8608
+ embedding_id = COALESCE(EXCLUDED.embedding_id, embedding_set_member.embedding_id),
8609
+ membership_type = EXCLUDED.membership_type,
8610
+ added_at = EXCLUDED.added_at,
8611
+ added_by = EXCLUDED.added_by`,
8612
+ [
8613
+ member.embedding_set_id,
8614
+ member.note_id,
8615
+ embeddingId,
8616
+ member.membership_type ?? "materialized",
8617
+ member.added_at ?? manifest.created_at,
8618
+ member.added_by ?? null
8619
+ ]
7681
8620
  );
7682
8621
  counts.embedding_set_members++;
7683
8622
  report?.({ phase: "embedding_set_members", done: index + 1, total: parsedEmbMembers.length });
@@ -7695,6 +8634,17 @@ async function importShard(db, data, options) {
7695
8634
  duration_ms: performance.now() - start
7696
8635
  };
7697
8636
  }
8637
+ if (options?.blobStore && blobsToHydrate.size > 0) {
8638
+ try {
8639
+ for (const [hash, bytes] of blobsToHydrate) {
8640
+ await options.blobStore.write(hash, bytes);
8641
+ }
8642
+ } catch (err) {
8643
+ warnings.push(
8644
+ `Imported metadata successfully but failed to hydrate ${blobsToHydrate.size} attachment blob(s) into the BlobStore: ${err instanceof Error ? err.message : String(err)}.`
8645
+ );
8646
+ }
8647
+ }
7698
8648
  return {
7699
8649
  success: true,
7700
8650
  counts,
@@ -7704,31 +8654,667 @@ async function importShard(db, data, options) {
7704
8654
  duration_ms: performance.now() - start
7705
8655
  };
7706
8656
  }
7707
- function parseJsonl(data) {
7708
- if (!data || data.byteLength === 0) return [];
7709
- const text = decoder.decode(data);
7710
- return text.split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
8657
+ async function resolveEmbeddingSetIdForServerEmbedding(db, model, vector) {
8658
+ const dimension = vectorDimension(vector);
8659
+ const existing = await db.query(
8660
+ `SELECT id FROM embedding_set
8661
+ WHERE model_name = $1 AND dimensions = $2
8662
+ ORDER BY created_at, id
8663
+ LIMIT 1`,
8664
+ [model, dimension]
8665
+ );
8666
+ if (existing.rows[0]?.id) return existing.rows[0].id;
8667
+ const id = generateId();
8668
+ await db.query(
8669
+ `INSERT INTO embedding_set (
8670
+ id, name, slug, description, purpose, document_count, embedding_count,
8671
+ is_system, keywords_json, model_name, dimensions, kind, created_at, updated_at
8672
+ ) VALUES ($1, $2, $3, NULL, NULL, 0, 0, false, '[]'::jsonb, $2, $4, 'physical', now(), now())`,
8673
+ [id, model, slugifyServerEmbeddingSet(model), dimension]
8674
+ );
8675
+ return id;
7711
8676
  }
7712
- function parseJsonArray(data) {
7713
- if (!data || data.byteLength === 0) return [];
7714
- return JSON.parse(decoder.decode(data));
8677
+ function vectorDimension(vector) {
8678
+ const inner = vector.replace(/^\[/, "").replace(/\]$/, "").trim();
8679
+ if (!inner) return 0;
8680
+ return inner.split(",").length;
8681
+ }
8682
+ function slugifyServerEmbeddingSet(value) {
8683
+ const slug = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
8684
+ return slug || "embedding-set";
8685
+ }
8686
+
8687
+ // schemas/knowledge-shard.schema.json
8688
+ var knowledge_shard_schema_default = {
8689
+ $schema: "https://json-schema.org/draft/2020-12/schema",
8690
+ $id: "https://fortemi.dev/schemas/knowledge-shard.schema.json",
8691
+ title: "Knowledge Shard (matric-shard) conformance schema",
8692
+ description: "Structural authority for the Knowledge Shard interchange contract shared between the Fortemi server (GET /api/v1/backup/knowledge-shard) and @fortemi/core. Covers the manifest, server-owned shard components, and React extension components tracked by issue #255.",
8693
+ type: "object",
8694
+ $defs: {
8695
+ isoDateTime: {
8696
+ type: "string",
8697
+ description: "ISO 8601 timestamp",
8698
+ minLength: 1
8699
+ },
8700
+ jsonObject: {
8701
+ type: "object",
8702
+ additionalProperties: true
8703
+ },
8704
+ nullableJsonObject: {
8705
+ type: ["object", "null"],
8706
+ additionalProperties: true
8707
+ },
8708
+ stringArray: {
8709
+ type: "array",
8710
+ items: { type: "string" }
8711
+ },
8712
+ manifest: {
8713
+ type: "object",
8714
+ additionalProperties: false,
8715
+ required: [
8716
+ "version",
8717
+ "matric_version",
8718
+ "format",
8719
+ "created_at",
8720
+ "components",
8721
+ "counts",
8722
+ "checksums",
8723
+ "min_reader_version"
8724
+ ],
8725
+ properties: {
8726
+ version: { type: "string", minLength: 1 },
8727
+ matric_version: { type: "string", minLength: 1 },
8728
+ format: { const: "matric-shard" },
8729
+ created_at: { $ref: "#/$defs/isoDateTime" },
8730
+ components: {
8731
+ type: "array",
8732
+ items: {
8733
+ enum: [
8734
+ "notes",
8735
+ "collections",
8736
+ "tags",
8737
+ "templates",
8738
+ "links",
8739
+ "embedding_sets",
8740
+ "embedding_configs",
8741
+ "embedding_set_members",
8742
+ "embeddings",
8743
+ "skos_schemes",
8744
+ "skos_concepts",
8745
+ "skos_relations",
8746
+ "note_skos_tags",
8747
+ "provenance_edges",
8748
+ "community_assignments",
8749
+ "communities",
8750
+ "graph_edges",
8751
+ "graph_sources"
8752
+ ]
8753
+ }
8754
+ },
8755
+ counts: {
8756
+ type: "object",
8757
+ additionalProperties: { type: "integer", minimum: 0 }
8758
+ },
8759
+ checksums: {
8760
+ type: "object",
8761
+ description: "filename -> sha256 hex",
8762
+ additionalProperties: { type: "string", pattern: "^[0-9a-f]{64}$" }
8763
+ },
8764
+ min_reader_version: { type: "string", minLength: 1 },
8765
+ migrated_from: {
8766
+ type: ["string", "null"],
8767
+ minLength: 1
8768
+ },
8769
+ migration_history: {
8770
+ type: "array",
8771
+ items: {
8772
+ type: "object",
8773
+ additionalProperties: false,
8774
+ required: [
8775
+ "from_version",
8776
+ "to_version",
8777
+ "migrated_at",
8778
+ "migrated_by",
8779
+ "changes"
8780
+ ],
8781
+ properties: {
8782
+ from_version: { type: "string", minLength: 1 },
8783
+ to_version: { type: "string", minLength: 1 },
8784
+ migrated_at: { $ref: "#/$defs/isoDateTime" },
8785
+ migrated_by: { type: "string", minLength: 1 },
8786
+ changes: { $ref: "#/$defs/stringArray" }
8787
+ }
8788
+ }
8789
+ },
8790
+ layout: { type: "object" }
8791
+ }
8792
+ },
8793
+ attachmentReference: {
8794
+ type: "object",
8795
+ additionalProperties: false,
8796
+ required: ["id", "path", "mime", "checksum", "bytes"],
8797
+ properties: {
8798
+ id: { type: "string", minLength: 1 },
8799
+ path: { type: "string", minLength: 1 },
8800
+ mime: { type: ["string", "null"] },
8801
+ checksum: { type: "string", minLength: 1 },
8802
+ bytes: { type: "integer", minimum: 0 }
8803
+ }
8804
+ },
8805
+ attachmentProjection: {
8806
+ type: "object",
8807
+ additionalProperties: false,
8808
+ required: ["extracted_text", "attachment"],
8809
+ properties: {
8810
+ extracted_text: { type: ["string", "null"] },
8811
+ attachment: { $ref: "#/$defs/attachmentReference" }
8812
+ }
8813
+ },
8814
+ note: {
8815
+ type: "object",
8816
+ additionalProperties: false,
8817
+ description: "One row of the shard `notes` component. Field names are the server shard contract, not the PGlite table column names.",
8818
+ required: [
8819
+ "id",
8820
+ "title",
8821
+ "original_content",
8822
+ "revised_content",
8823
+ "format",
8824
+ "source",
8825
+ "starred",
8826
+ "archived",
8827
+ "tags",
8828
+ "created_at",
8829
+ "updated_at"
8830
+ ],
8831
+ properties: {
8832
+ id: { type: "string", minLength: 1 },
8833
+ title: { type: ["string", "null"] },
8834
+ original_content: { type: "string" },
8835
+ revised_content: { type: ["string", "null"] },
8836
+ collection_id: { type: ["string", "null"] },
8837
+ attachments: {
8838
+ type: "array",
8839
+ items: { $ref: "#/$defs/attachmentProjection" }
8840
+ },
8841
+ format: { type: "string" },
8842
+ source: { type: "string" },
8843
+ starred: { type: "boolean" },
8844
+ archived: { type: "boolean" },
8845
+ tags: { type: "array", items: { type: "string" } },
8846
+ created_at: { $ref: "#/$defs/isoDateTime" },
8847
+ updated_at: { $ref: "#/$defs/isoDateTime" },
8848
+ deleted_at: { type: ["string", "null"] }
8849
+ }
8850
+ },
8851
+ collection: {
8852
+ type: "object",
8853
+ additionalProperties: false,
8854
+ required: ["id", "name", "description", "parent_id", "created_at"],
8855
+ properties: {
8856
+ id: { type: "string", minLength: 1 },
8857
+ name: { type: "string" },
8858
+ description: { type: ["string", "null"] },
8859
+ parent_id: { type: ["string", "null"] },
8860
+ created_at: { $ref: "#/$defs/isoDateTime" },
8861
+ note_count: { type: "integer", minimum: 0 }
8862
+ }
8863
+ },
8864
+ tag: {
8865
+ type: "object",
8866
+ additionalProperties: false,
8867
+ required: ["name", "created_at"],
8868
+ properties: {
8869
+ name: { type: "string", minLength: 1 },
8870
+ created_at: { $ref: "#/$defs/isoDateTime" }
8871
+ }
8872
+ },
8873
+ template: {
8874
+ type: "object",
8875
+ additionalProperties: false,
8876
+ required: ["id", "name", "description", "content", "format", "default_tags", "collection_id", "created_at", "updated_at"],
8877
+ properties: {
8878
+ id: { type: "string", minLength: 1 },
8879
+ name: { type: "string" },
8880
+ description: { type: ["string", "null"] },
8881
+ content: { type: "string" },
8882
+ format: { type: "string" },
8883
+ default_tags: { $ref: "#/$defs/stringArray" },
8884
+ collection_id: { type: ["string", "null"] },
8885
+ created_at: { $ref: "#/$defs/isoDateTime" },
8886
+ updated_at: { $ref: "#/$defs/isoDateTime" }
8887
+ }
8888
+ },
8889
+ link: {
8890
+ type: "object",
8891
+ additionalProperties: false,
8892
+ required: ["id", "from_note_id", "to_note_id", "to_url", "kind", "score", "created_at", "metadata"],
8893
+ properties: {
8894
+ id: { type: "string", minLength: 1 },
8895
+ from_note_id: { type: "string", minLength: 1 },
8896
+ to_note_id: { type: ["string", "null"] },
8897
+ to_url: { type: ["string", "null"] },
8898
+ kind: { type: "string" },
8899
+ score: { type: ["number", "null"] },
8900
+ created_at: { $ref: "#/$defs/isoDateTime" },
8901
+ metadata: { $ref: "#/$defs/nullableJsonObject" }
8902
+ }
8903
+ },
8904
+ embeddingSet: {
8905
+ type: "object",
8906
+ additionalProperties: false,
8907
+ required: ["id", "name", "slug", "description", "purpose", "document_count", "embedding_count", "is_system", "keywords", "model", "dimension"],
8908
+ properties: {
8909
+ id: { type: "string", minLength: 1 },
8910
+ name: { type: "string" },
8911
+ slug: { type: ["string", "null"] },
8912
+ description: { type: ["string", "null"] },
8913
+ purpose: { type: ["string", "null"] },
8914
+ document_count: { type: "integer", minimum: 0 },
8915
+ embedding_count: { type: "integer", minimum: 0 },
8916
+ is_system: { type: "boolean" },
8917
+ keywords: { $ref: "#/$defs/stringArray" },
8918
+ model: { type: "string" },
8919
+ dimension: { type: "integer", minimum: 1 },
8920
+ kind: { enum: ["physical", "filter", "virtual"] },
8921
+ mode: { type: ["string", "null"] },
8922
+ truncate_dimension: { type: ["integer", "null"], minimum: 1 },
8923
+ criteria: { $ref: "#/$defs/nullableJsonObject" },
8924
+ source: { $ref: "#/$defs/nullableJsonObject" },
8925
+ compatibility: { $ref: "#/$defs/nullableJsonObject" },
8926
+ materialization: { $ref: "#/$defs/nullableJsonObject" },
8927
+ freshness: { $ref: "#/$defs/nullableJsonObject" },
8928
+ created_at: { $ref: "#/$defs/isoDateTime" },
8929
+ updated_at: { $ref: "#/$defs/isoDateTime" }
8930
+ }
8931
+ },
8932
+ embeddingSetMember: {
8933
+ type: "object",
8934
+ additionalProperties: false,
8935
+ required: ["embedding_set_id", "note_id", "membership_type", "added_at", "added_by"],
8936
+ properties: {
8937
+ embedding_set_id: { type: "string", minLength: 1 },
8938
+ note_id: { type: "string", minLength: 1 },
8939
+ membership_type: { type: "string" },
8940
+ added_at: { $ref: "#/$defs/isoDateTime" },
8941
+ added_by: { type: ["string", "null"] }
8942
+ }
8943
+ },
8944
+ embeddingConfig: {
8945
+ type: "object",
8946
+ additionalProperties: false,
8947
+ required: ["id", "name", "description", "model", "dimension", "chunk_size", "chunk_overlap", "is_default"],
8948
+ properties: {
8949
+ id: { type: "string", minLength: 1 },
8950
+ name: { type: "string" },
8951
+ description: { type: ["string", "null"] },
8952
+ model: { type: "string" },
8953
+ dimension: { type: "integer", minimum: 1 },
8954
+ chunk_size: { type: "integer", minimum: 1 },
8955
+ chunk_overlap: { type: "integer", minimum: 0 },
8956
+ is_default: { type: "boolean" }
8957
+ }
8958
+ },
8959
+ embedding: {
8960
+ type: "object",
8961
+ additionalProperties: false,
8962
+ required: ["id", "note_id", "chunk_index", "text", "vector", "model"],
8963
+ properties: {
8964
+ id: { type: "string", minLength: 1 },
8965
+ note_id: { type: "string", minLength: 1 },
8966
+ chunk_index: { type: "integer", minimum: 0 },
8967
+ text: { type: "string" },
8968
+ vector: { type: "array", items: { type: "number" } },
8969
+ model: { type: "string" },
8970
+ embedding_set_id: { type: "string", minLength: 1 },
8971
+ created_at: { $ref: "#/$defs/isoDateTime" }
8972
+ }
8973
+ },
8974
+ skosScheme: {
8975
+ type: "object",
8976
+ additionalProperties: false,
8977
+ required: ["id", "title", "description", "created_at", "updated_at"],
8978
+ properties: {
8979
+ id: { type: "string", minLength: 1 },
8980
+ title: { type: "string" },
8981
+ description: { type: ["string", "null"] },
8982
+ created_at: { $ref: "#/$defs/isoDateTime" },
8983
+ updated_at: { $ref: "#/$defs/isoDateTime" }
8984
+ }
8985
+ },
8986
+ skosConcept: {
8987
+ type: "object",
8988
+ additionalProperties: false,
8989
+ required: ["id", "scheme_id", "pref_label", "alt_labels", "definition", "created_at", "updated_at"],
8990
+ properties: {
8991
+ id: { type: "string", minLength: 1 },
8992
+ scheme_id: { type: "string", minLength: 1 },
8993
+ pref_label: { type: "string" },
8994
+ alt_labels: { $ref: "#/$defs/stringArray" },
8995
+ definition: { type: ["string", "null"] },
8996
+ created_at: { $ref: "#/$defs/isoDateTime" },
8997
+ updated_at: { $ref: "#/$defs/isoDateTime" }
8998
+ }
8999
+ },
9000
+ skosRelation: {
9001
+ type: "object",
9002
+ additionalProperties: false,
9003
+ required: ["id", "source_concept_id", "target_concept_id", "relation_type", "created_at"],
9004
+ properties: {
9005
+ id: { type: "string", minLength: 1 },
9006
+ source_concept_id: { type: "string", minLength: 1 },
9007
+ target_concept_id: { type: "string", minLength: 1 },
9008
+ relation_type: { enum: ["broader", "narrower", "related"] },
9009
+ created_at: { $ref: "#/$defs/isoDateTime" }
9010
+ }
9011
+ },
9012
+ noteSkosTag: {
9013
+ type: "object",
9014
+ additionalProperties: false,
9015
+ required: ["id", "note_id", "concept_id", "created_at"],
9016
+ properties: {
9017
+ id: { type: "string", minLength: 1 },
9018
+ note_id: { type: "string", minLength: 1 },
9019
+ concept_id: { type: "string", minLength: 1 },
9020
+ created_at: { $ref: "#/$defs/isoDateTime" }
9021
+ }
9022
+ },
9023
+ provenanceEdge: {
9024
+ type: "object",
9025
+ additionalProperties: false,
9026
+ required: ["id", "entity_type", "entity_id", "activity", "agent", "started_at", "ended_at", "attributes"],
9027
+ properties: {
9028
+ id: { type: "string", minLength: 1 },
9029
+ entity_type: { type: "string" },
9030
+ entity_id: { type: "string", minLength: 1 },
9031
+ activity: { type: "string" },
9032
+ agent: { type: "string" },
9033
+ started_at: { $ref: "#/$defs/isoDateTime" },
9034
+ ended_at: { type: ["string", "null"] },
9035
+ attributes: { $ref: "#/$defs/nullableJsonObject" }
9036
+ }
9037
+ },
9038
+ artifactFreshness: {
9039
+ type: "object",
9040
+ additionalProperties: false,
9041
+ required: ["status"],
9042
+ properties: {
9043
+ status: { enum: ["fresh", "stale", "unknown"] },
9044
+ checked_at: { $ref: "#/$defs/isoDateTime" },
9045
+ stale_reason: { type: "string" },
9046
+ source_hashes: { $ref: "#/$defs/jsonObject" }
9047
+ }
9048
+ },
9049
+ graphSource: {
9050
+ type: "object",
9051
+ additionalProperties: false,
9052
+ required: ["id", "name", "kind", "input_hash", "freshness", "created_at"],
9053
+ properties: {
9054
+ id: { type: "string", minLength: 1 },
9055
+ name: { type: "string" },
9056
+ kind: { enum: ["link", "similarity", "search", "manual", "imported"] },
9057
+ source_table: { type: ["string", "null"] },
9058
+ embedding_set_id: { type: ["string", "null"] },
9059
+ virtual_set_id: { type: ["string", "null"] },
9060
+ model: { type: ["string", "null"] },
9061
+ dimension: { type: ["integer", "null"], minimum: 1 },
9062
+ truncate_dimension: { type: ["integer", "null"], minimum: 1 },
9063
+ metric: { type: ["string", "null"] },
9064
+ algorithm: { type: ["string", "null"] },
9065
+ parameters: { $ref: "#/$defs/jsonObject" },
9066
+ input_hash: { type: "string" },
9067
+ freshness: { $ref: "#/$defs/artifactFreshness" },
9068
+ created_at: { $ref: "#/$defs/isoDateTime" }
9069
+ }
9070
+ },
9071
+ graphEdge: {
9072
+ type: "object",
9073
+ additionalProperties: false,
9074
+ required: ["graph_source_id", "from_note_id", "to_note_id", "weight", "kind"],
9075
+ properties: {
9076
+ graph_source_id: { type: "string", minLength: 1 },
9077
+ from_note_id: { type: "string", minLength: 1 },
9078
+ to_note_id: { type: "string", minLength: 1 },
9079
+ weight: { type: "number" },
9080
+ kind: { enum: ["link", "similarity", "manual"] },
9081
+ rank: { type: ["integer", "null"], minimum: 0 },
9082
+ metadata: { $ref: "#/$defs/jsonObject" }
9083
+ }
9084
+ },
9085
+ community: {
9086
+ type: "object",
9087
+ additionalProperties: false,
9088
+ required: ["id"],
9089
+ properties: {
9090
+ id: { type: "string", minLength: 1 },
9091
+ label: { type: ["string", "null"] },
9092
+ rank: { type: ["integer", "null"], minimum: 0 },
9093
+ size: { type: ["integer", "null"], minimum: 0 },
9094
+ confidence: { type: ["number", "null"] },
9095
+ representative_note_ids: { $ref: "#/$defs/stringArray" },
9096
+ metadata: { $ref: "#/$defs/jsonObject" }
9097
+ }
9098
+ },
9099
+ communitySet: {
9100
+ type: "object",
9101
+ additionalProperties: false,
9102
+ required: ["id", "graph_source_id", "name", "source_type", "input_hash", "freshness", "communities", "created_at"],
9103
+ properties: {
9104
+ id: { type: "string", minLength: 1 },
9105
+ graph_source_id: { type: "string", minLength: 1 },
9106
+ name: { type: "string" },
9107
+ source_type: { enum: ["precomputed", "dynamic-snapshot", "user-authored", "imported"] },
9108
+ algorithm: { type: ["string", "null"] },
9109
+ parameters: { $ref: "#/$defs/jsonObject" },
9110
+ input_hash: { type: "string" },
9111
+ freshness: { $ref: "#/$defs/artifactFreshness" },
9112
+ communities: {
9113
+ type: "array",
9114
+ items: { $ref: "#/$defs/community" }
9115
+ },
9116
+ created_at: { $ref: "#/$defs/isoDateTime" }
9117
+ }
9118
+ },
9119
+ communityAssignment: {
9120
+ type: "object",
9121
+ additionalProperties: false,
9122
+ required: ["community_set_id", "community_id", "note_id", "source_type"],
9123
+ properties: {
9124
+ community_set_id: { type: "string", minLength: 1 },
9125
+ community_id: { type: "string", minLength: 1 },
9126
+ note_id: { type: "string", minLength: 1 },
9127
+ confidence: { type: ["number", "null"] },
9128
+ source_type: { enum: ["precomputed", "dynamic-snapshot", "user-authored", "imported"] },
9129
+ metadata: { $ref: "#/$defs/jsonObject" }
9130
+ }
9131
+ }
9132
+ }
9133
+ };
9134
+
9135
+ // src/shard/schema-validator.ts
9136
+ var decoder3 = new TextDecoder();
9137
+ var COMPONENT_SCHEMA_DEFS = {
9138
+ notes: "note",
9139
+ collections: "collection",
9140
+ tags: "tag",
9141
+ templates: "template",
9142
+ links: "link",
9143
+ embedding_sets: "embeddingSet",
9144
+ embedding_set_members: "embeddingSetMember",
9145
+ embedding_configs: "embeddingConfig",
9146
+ embeddings: "embedding",
9147
+ skos_schemes: "skosScheme",
9148
+ skos_concepts: "skosConcept",
9149
+ skos_relations: "skosRelation",
9150
+ note_skos_tags: "noteSkosTag",
9151
+ provenance_edges: "provenanceEdge",
9152
+ graph_sources: "graphSource",
9153
+ graph_edges: "graphEdge",
9154
+ communities: "communitySet",
9155
+ community_assignments: "communityAssignment"
9156
+ };
9157
+ var COMPONENT_FILES = {
9158
+ notes: { file: "notes.jsonl", encoding: "jsonl" },
9159
+ collections: { file: "collections.json", encoding: "json-array" },
9160
+ tags: { file: "tags.json", encoding: "json-array" },
9161
+ templates: { file: "templates.json", encoding: "json-array" },
9162
+ links: { file: "links.jsonl", encoding: "jsonl" },
9163
+ embedding_sets: { file: "embedding_sets.json", encoding: "json-array" },
9164
+ embedding_set_members: { file: "embedding_set_members.jsonl", encoding: "jsonl" },
9165
+ embedding_configs: { file: "embedding_configs.json", encoding: "json-array" },
9166
+ embeddings: { file: "embeddings.jsonl", encoding: "jsonl" },
9167
+ skos_schemes: { file: "skos_schemes.json", encoding: "json-array" },
9168
+ skos_concepts: { file: "skos_concepts.json", encoding: "json-array" },
9169
+ skos_relations: { file: "skos_relations.jsonl", encoding: "jsonl" },
9170
+ note_skos_tags: { file: "note_skos_tags.jsonl", encoding: "jsonl" },
9171
+ provenance_edges: { file: "provenance_edges.jsonl", encoding: "jsonl" },
9172
+ graph_sources: { file: "graph_sources.json", encoding: "json-array" },
9173
+ graph_edges: { file: "graph_edges.jsonl", encoding: "jsonl" },
9174
+ communities: { file: "communities.json", encoding: "json-array" },
9175
+ community_assignments: { file: "community_assignments.jsonl", encoding: "jsonl" }
9176
+ };
9177
+ var ajvInstance;
9178
+ var validators = /* @__PURE__ */ new Map();
9179
+ function getKnowledgeShardSchema() {
9180
+ return knowledge_shard_schema_default;
9181
+ }
9182
+ function getAjv() {
9183
+ if (!ajvInstance) {
9184
+ ajvInstance = new Ajv2020({
9185
+ allErrors: true,
9186
+ strict: true,
9187
+ validateFormats: false
9188
+ });
9189
+ ajvInstance.addSchema(knowledge_shard_schema_default);
9190
+ }
9191
+ return ajvInstance;
9192
+ }
9193
+ function validatorFor(defName) {
9194
+ const cached = validators.get(defName);
9195
+ if (cached) return cached;
9196
+ const validator = getAjv().getSchema(`${knowledge_shard_schema_default.$id}#/$defs/${defName}`) ?? getAjv().compile({ $ref: `${knowledge_shard_schema_default.$id}#/$defs/${defName}` });
9197
+ validators.set(defName, validator);
9198
+ return validator;
9199
+ }
9200
+ function formatErrors(errors) {
9201
+ return (errors ?? []).map((error) => {
9202
+ const path = error.instancePath || "(root)";
9203
+ return `${path} ${error.message ?? "is invalid"}`;
9204
+ });
9205
+ }
9206
+ function validateShardManifest(value) {
9207
+ const validate = validatorFor("manifest");
9208
+ const valid = validate(value);
9209
+ return { valid, errors: formatErrors(validate.errors) };
9210
+ }
9211
+ function parseJsonArray(bytes, path) {
9212
+ if (!bytes) return { records: [], errors: [] };
9213
+ try {
9214
+ const value = JSON.parse(decoder3.decode(bytes));
9215
+ if (!Array.isArray(value)) return { records: [], errors: [`${path} must be a JSON array`] };
9216
+ return { records: value, errors: [] };
9217
+ } catch (error) {
9218
+ return { records: [], errors: [`${path} failed to parse: ${error instanceof Error ? error.message : String(error)}`] };
9219
+ }
9220
+ }
9221
+ function parseJsonl(bytes, path) {
9222
+ if (!bytes) return { records: [], errors: [] };
9223
+ const text = decoder3.decode(bytes).trim();
9224
+ if (!text) return { records: [], errors: [] };
9225
+ const records = [];
9226
+ const errors = [];
9227
+ for (const [index, line] of text.split("\n").entries()) {
9228
+ try {
9229
+ records.push(JSON.parse(line));
9230
+ } catch (error) {
9231
+ errors.push(`${path}:${index + 1} failed to parse: ${error instanceof Error ? error.message : String(error)}`);
9232
+ }
9233
+ }
9234
+ return { records, errors };
9235
+ }
9236
+ function unpackShardFiles(input) {
9237
+ if (input instanceof Map) return input;
9238
+ return unpackTarGz(input instanceof ArrayBuffer ? new Uint8Array(input) : input);
9239
+ }
9240
+ function validateComponentRecords(component, records, path) {
9241
+ const errors = [];
9242
+ for (const [index, record] of records.entries()) {
9243
+ const result = validateShardComponentRecord(component, record);
9244
+ if (!result.valid) {
9245
+ errors.push(...result.errors.map((error) => `${path}[${index}] ${error}`));
9246
+ }
9247
+ }
9248
+ return errors;
9249
+ }
9250
+ function validateShardArchive(input) {
9251
+ let files;
9252
+ try {
9253
+ files = unpackShardFiles(input);
9254
+ } catch (error) {
9255
+ return {
9256
+ valid: false,
9257
+ errors: [`archive failed to unpack: ${error instanceof Error ? error.message : String(error)}`]
9258
+ };
9259
+ }
9260
+ const manifestBytes = files.get("manifest.json");
9261
+ if (!manifestBytes) return { valid: false, errors: ["manifest.json is missing"] };
9262
+ let manifest;
9263
+ try {
9264
+ manifest = JSON.parse(decoder3.decode(manifestBytes));
9265
+ } catch (error) {
9266
+ return {
9267
+ valid: false,
9268
+ errors: [`manifest.json failed to parse: ${error instanceof Error ? error.message : String(error)}`]
9269
+ };
9270
+ }
9271
+ const errors = [];
9272
+ const manifestResult = validateShardManifest(manifest);
9273
+ if (!manifestResult.valid) {
9274
+ errors.push(...manifestResult.errors.map((error) => `manifest.json ${error}`));
9275
+ }
9276
+ const componentsToValidate = new Set(manifest.components);
9277
+ for (const [component, spec] of Object.entries(COMPONENT_FILES)) {
9278
+ if (files.has(spec.file)) componentsToValidate.add(component);
9279
+ }
9280
+ for (const component of componentsToValidate) {
9281
+ const spec = COMPONENT_FILES[component];
9282
+ if (!spec) continue;
9283
+ if (component === "notes" && manifest.layout?.clusters?.notes?.length) {
9284
+ for (const cluster of manifest.layout.clusters.notes) {
9285
+ const parsed2 = parseJsonl(files.get(cluster.href), cluster.href);
9286
+ errors.push(...parsed2.errors);
9287
+ errors.push(...validateComponentRecords("notes", parsed2.records, cluster.href));
9288
+ }
9289
+ continue;
9290
+ }
9291
+ const parsed = spec.encoding === "json-array" ? parseJsonArray(files.get(spec.file), spec.file) : parseJsonl(files.get(spec.file), spec.file);
9292
+ errors.push(...parsed.errors);
9293
+ errors.push(...validateComponentRecords(component, parsed.records, spec.file));
9294
+ }
9295
+ return { valid: errors.length === 0, errors };
9296
+ }
9297
+ function validateShardComponentRecord(component, value) {
9298
+ const defName = COMPONENT_SCHEMA_DEFS[component];
9299
+ const validate = validatorFor(defName);
9300
+ const valid = validate(value);
9301
+ return { valid, errors: formatErrors(validate.errors) };
9302
+ }
9303
+ function assertShardComponentRecord(component, value) {
9304
+ const result = validateShardComponentRecord(component, value);
9305
+ if (!result.valid) {
9306
+ throw new Error(`Invalid shard ${component} record:
9307
+ ${result.errors.join("\n")}`);
9308
+ }
7715
9309
  }
7716
9310
 
7717
9311
  // src/shard/shard-reader.ts
7718
- var decoder2 = new TextDecoder();
9312
+ var decoder4 = new TextDecoder();
7719
9313
  function assertSafeComponentName(filename) {
7720
9314
  if (filename.length === 0 || filename.startsWith("/") || filename.includes("\\") || filename.includes("\0") || filename.includes(":") || filename.split("/").some((segment) => segment === "..")) {
7721
9315
  throw new Error(`Refusing to read unsafe shard component path: ${JSON.stringify(filename)}`);
7722
9316
  }
7723
9317
  }
7724
- function parseJsonlBytes(data) {
7725
- if (!data || data.byteLength === 0) return [];
7726
- return decoder2.decode(data).split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
7727
- }
7728
- function parseJsonArrayBytes(data) {
7729
- if (!data || data.byteLength === 0) return [];
7730
- return JSON.parse(decoder2.decode(data));
7731
- }
7732
9318
  var DEFAULT_WEIGHTS = { title: 4, content: 1, tag: 2 };
7733
9319
  async function toBytes(blobOrBytes) {
7734
9320
  if (blobOrBytes instanceof Uint8Array) return blobOrBytes;
@@ -7741,8 +9327,13 @@ var PackedComponentStore = class {
7741
9327
  this.files = files;
7742
9328
  this.manifest = manifest;
7743
9329
  }
7744
- read(filename) {
7745
- return Promise.resolve(this.files.get(filename));
9330
+ async read(filename) {
9331
+ const bytes = this.files.get(filename);
9332
+ const expectedChecksum = this.manifest.checksums?.[filename];
9333
+ if (bytes && expectedChecksum && await sha256Hex(bytes) !== expectedChecksum) {
9334
+ throw new Error(`Checksum validation failed for shard component: ${filename}`);
9335
+ }
9336
+ return bytes;
7746
9337
  }
7747
9338
  };
7748
9339
  var UrlComponentStore = class {
@@ -7750,10 +9341,12 @@ var UrlComponentStore = class {
7750
9341
  baseUrl;
7751
9342
  fetchImpl;
7752
9343
  cache = /* @__PURE__ */ new Map();
7753
- constructor(baseUrl, fetchImpl, manifest) {
9344
+ maxComponentBytes;
9345
+ constructor(baseUrl, fetchImpl, manifest, maxComponentBytes) {
7754
9346
  this.baseUrl = baseUrl.replace(/\/$/, "");
7755
9347
  this.fetchImpl = fetchImpl;
7756
9348
  this.manifest = manifest;
9349
+ this.maxComponentBytes = maxComponentBytes;
7757
9350
  }
7758
9351
  async read(filename) {
7759
9352
  assertSafeComponentName(filename);
@@ -7763,12 +9356,47 @@ var UrlComponentStore = class {
7763
9356
  this.cache.set(filename, void 0);
7764
9357
  return void 0;
7765
9358
  }
7766
- const bytes = new Uint8Array(await response.arrayBuffer());
9359
+ const declaredLength = Number(response.headers.get("content-length"));
9360
+ if (Number.isFinite(declaredLength) && declaredLength > this.maxComponentBytes) {
9361
+ throw new Error(`Shard component ${filename} exceeds cap ${this.maxComponentBytes} bytes`);
9362
+ }
9363
+ const chunks = [];
9364
+ let total = 0;
9365
+ if (response.body) {
9366
+ const reader = response.body.getReader();
9367
+ while (true) {
9368
+ const { done, value } = await reader.read();
9369
+ if (done) break;
9370
+ total += value.byteLength;
9371
+ if (total > this.maxComponentBytes) {
9372
+ await reader.cancel();
9373
+ throw new Error(`Shard component ${filename} exceeds cap ${this.maxComponentBytes} bytes`);
9374
+ }
9375
+ chunks.push(value);
9376
+ }
9377
+ } else {
9378
+ const value = new Uint8Array(await response.arrayBuffer());
9379
+ total = value.byteLength;
9380
+ if (total > this.maxComponentBytes) {
9381
+ throw new Error(`Shard component ${filename} exceeds cap ${this.maxComponentBytes} bytes`);
9382
+ }
9383
+ chunks.push(value);
9384
+ }
9385
+ const bytes = new Uint8Array(total);
9386
+ let offset = 0;
9387
+ for (const chunk of chunks) {
9388
+ bytes.set(chunk, offset);
9389
+ offset += chunk.byteLength;
9390
+ }
9391
+ const expectedChecksum = this.manifest.checksums?.[filename];
9392
+ if (expectedChecksum && await sha256Hex(bytes) !== expectedChecksum) {
9393
+ throw new Error(`Checksum validation failed for shard component: ${filename}`);
9394
+ }
7767
9395
  this.cache.set(filename, bytes);
7768
9396
  return bytes;
7769
9397
  }
7770
9398
  };
7771
- async function resolveStore(source) {
9399
+ async function resolveStore(source, maxComponentBytes) {
7772
9400
  if (typeof source === "object" && "baseUrl" in source) {
7773
9401
  const fetchImpl = source.fetchImpl ?? globalThis.fetch;
7774
9402
  const base = source.baseUrl.replace(/\/$/, "");
@@ -7777,20 +9405,20 @@ async function resolveStore(source) {
7777
9405
  throw new Error(`Failed to fetch shard manifest (${manifestResponse.status}): ${base}/manifest.json`);
7778
9406
  }
7779
9407
  const manifest2 = await manifestResponse.json();
7780
- return new UrlComponentStore(base, fetchImpl, manifest2);
9408
+ return new UrlComponentStore(base, fetchImpl, manifest2, maxComponentBytes);
7781
9409
  }
7782
9410
  const bytes = await toBytes(source);
7783
9411
  const files = unpackTarGz(bytes);
7784
9412
  const manifestBytes = files.get("manifest.json");
7785
9413
  if (!manifestBytes) throw new Error("Missing manifest.json in shard archive");
7786
- const manifest = JSON.parse(decoder2.decode(manifestBytes));
9414
+ const manifest = JSON.parse(decoder4.decode(manifestBytes));
7787
9415
  return new PackedComponentStore(files, manifest);
7788
9416
  }
7789
9417
  function tokenize(query) {
7790
9418
  return query.toLowerCase().split(/[^a-z0-9]+/i).filter((token) => token.length > 0);
7791
9419
  }
7792
9420
  function noteSearchText(note) {
7793
- const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
9421
+ const extractedText = (note.attachments ?? note.binary_sources)?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
7794
9422
  return `${note.title ?? ""} ${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.toLowerCase();
7795
9423
  }
7796
9424
  function countOccurrences(haystack, needle) {
@@ -7812,7 +9440,7 @@ function noteMatchesTokens(note, tokens) {
7812
9440
  function rankNote(note, tokens, weights) {
7813
9441
  if (tokens.length === 0) return 0;
7814
9442
  const title = (note.title ?? "").toLowerCase();
7815
- const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
9443
+ const extractedText = (note.attachments ?? note.binary_sources)?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
7816
9444
  const content = `${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.toLowerCase();
7817
9445
  const tagText = note.tags.join(" ").toLowerCase();
7818
9446
  let score = 0;
@@ -7824,7 +9452,7 @@ function rankNote(note, tokens, weights) {
7824
9452
  return score;
7825
9453
  }
7826
9454
  function makeSnippet(note, tokens, length) {
7827
- const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
9455
+ const extractedText = (note.attachments ?? note.binary_sources)?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
7828
9456
  const content = `${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.trim();
7829
9457
  if (tokens.length === 0) return content.slice(0, length);
7830
9458
  const lower = content.toLowerCase();
@@ -8058,8 +9686,8 @@ var ShardReaderImpl = class {
8058
9686
  }
8059
9687
  };
8060
9688
  async function openShard(source, options = {}) {
8061
- const store = await resolveStore(source);
8062
- if (store.manifest.min_reader_version && store.manifest.min_reader_version > CURRENT_SHARD_VERSION) {
9689
+ const store = await resolveStore(source, options.maxComponentBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES);
9690
+ if (store.manifest.min_reader_version && compareShardVersions(store.manifest.min_reader_version, CURRENT_SHARD_VERSION) > 0) {
8063
9691
  throw new Error(
8064
9692
  `Shard requires reader version ${store.manifest.min_reader_version}, but this build supports ${CURRENT_SHARD_VERSION}. Import the shard instead.`
8065
9693
  );
@@ -8068,7 +9696,7 @@ async function openShard(source, options = {}) {
8068
9696
  }
8069
9697
 
8070
9698
  // src/shard/semantic-providers.ts
8071
- var decoder3 = new TextDecoder();
9699
+ var decoder5 = new TextDecoder();
8072
9700
  function cosine(a, b) {
8073
9701
  let dot = 0;
8074
9702
  let normA = 0;
@@ -8092,7 +9720,7 @@ function createCosineSemanticProvider(options) {
8092
9720
  vectors = [];
8093
9721
  return;
8094
9722
  }
8095
- vectors = decoder3.decode(bytes).split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
9723
+ vectors = decoder5.decode(bytes).split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
8096
9724
  },
8097
9725
  async search(query, k) {
8098
9726
  const queryVector = await options.embedQuery(query);
@@ -8230,7 +9858,7 @@ var AIWG_SCAN_REQUIRED_FIELDS = [
8230
9858
  ];
8231
9859
  function isPrivacyExcluded(record, options) {
8232
9860
  const privacy = record.privacy;
8233
- if (!privacy) return false;
9861
+ if (!privacy || !isPrivacyClassification(privacy.classification) || typeof privacy.pii !== "boolean") return true;
8234
9862
  if (privacy.classification === "private" && !options?.includePrivate) return true;
8235
9863
  if (privacy.pii && !options?.includePii) return true;
8236
9864
  return false;
@@ -8299,6 +9927,10 @@ function validateOptionalRichMetadata(item, index, errors) {
8299
9927
  errors.push("items[" + index + "].skos_concepts must be an array when present");
8300
9928
  } else {
8301
9929
  for (const [conceptIndex, concept] of item.skos_concepts.entries()) {
9930
+ if (!isPlainRecord(concept)) {
9931
+ errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "] must be an object");
9932
+ continue;
9933
+ }
8302
9934
  if (!hasString(concept.id)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].id is required");
8303
9935
  if (!hasString(concept.prefLabel)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].prefLabel is required");
8304
9936
  if (!isOptionalStringArray(concept.altLabels)) errors.push("items[" + index + "].skos_concepts[" + conceptIndex + "].altLabels must be a string array");
@@ -8313,6 +9945,10 @@ function validateOptionalRichMetadata(item, index, errors) {
8313
9945
  errors.push("items[" + index + "].skos_relations must be an array when present");
8314
9946
  } else {
8315
9947
  for (const [relationIndex, relation] of item.skos_relations.entries()) {
9948
+ if (!isPlainRecord(relation)) {
9949
+ errors.push("items[" + index + "].skos_relations[" + relationIndex + "] must be an object");
9950
+ continue;
9951
+ }
8316
9952
  if (!hasString(relation.type)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].type is required");
8317
9953
  if (!hasString(relation.source_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].source_id is required");
8318
9954
  if (!hasString(relation.target_id)) errors.push("items[" + index + "].skos_relations[" + relationIndex + "].target_id is required");
@@ -8327,6 +9963,10 @@ function validateOptionalRichMetadata(item, index, errors) {
8327
9963
  errors.push("items[" + index + "].provenance_events must be an array when present");
8328
9964
  } else {
8329
9965
  for (const [eventIndex, event] of item.provenance_events.entries()) {
9966
+ if (!isPlainRecord(event)) {
9967
+ errors.push("items[" + index + "].provenance_events[" + eventIndex + "] must be an object");
9968
+ continue;
9969
+ }
8330
9970
  if (!hasString(event.activity)) errors.push("items[" + index + "].provenance_events[" + eventIndex + "].activity is required");
8331
9971
  if (event.attributes !== void 0 && !isPlainRecord(event.attributes)) {
8332
9972
  errors.push("items[" + index + "].provenance_events[" + eventIndex + "].attributes must be an object");
@@ -8336,6 +9976,10 @@ function validateOptionalRichMetadata(item, index, errors) {
8336
9976
  }
8337
9977
  if (Array.isArray(item.relationships)) {
8338
9978
  for (const [relationshipIndex, relationship] of item.relationships.entries()) {
9979
+ if (!isPlainRecord(relationship)) {
9980
+ errors.push("items[" + index + "].relationships[" + relationshipIndex + "] must be an object");
9981
+ continue;
9982
+ }
8339
9983
  if (relationship.metadata !== void 0 && !isPlainRecord(relationship.metadata)) {
8340
9984
  errors.push("items[" + index + "].relationships[" + relationshipIndex + "].metadata must be an object");
8341
9985
  }
@@ -8364,7 +10008,10 @@ function validateOptionalRichMetadata(item, index, errors) {
8364
10008
  errors.push("items[" + index + "].chunks must be an array when present");
8365
10009
  } else {
8366
10010
  for (const [chunkIndex, chunk] of item.chunks.entries()) {
8367
- if (!isPlainRecord(chunk)) errors.push("items[" + index + "].chunks[" + chunkIndex + "] must be an object");
10011
+ if (!isPlainRecord(chunk)) {
10012
+ errors.push("items[" + index + "].chunks[" + chunkIndex + "] must be an object");
10013
+ continue;
10014
+ }
8368
10015
  if (chunk.metadata !== void 0 && !isPlainRecord(chunk.metadata)) {
8369
10016
  errors.push("items[" + index + "].chunks[" + chunkIndex + "].metadata must be an object");
8370
10017
  }
@@ -8376,9 +10023,12 @@ function validateOptionalRichMetadata(item, index, errors) {
8376
10023
  errors.push("items[" + index + "].embeddings must be an array when present");
8377
10024
  } else {
8378
10025
  for (const [embeddingIndex, embedding] of item.embeddings.entries()) {
8379
- if (!isPlainRecord(embedding)) errors.push("items[" + index + "].embeddings[" + embeddingIndex + "] must be an object");
8380
- const vector2 = embedding.embedding ?? embedding.vector;
8381
- if (vector2 !== void 0 && (!Array.isArray(vector2) || !vector2.every((entry) => typeof entry === "number"))) {
10026
+ if (!isPlainRecord(embedding)) {
10027
+ errors.push("items[" + index + "].embeddings[" + embeddingIndex + "] must be an object");
10028
+ continue;
10029
+ }
10030
+ const vector = embedding.embedding ?? embedding.vector;
10031
+ if (vector !== void 0 && (!Array.isArray(vector) || !vector.every((entry) => typeof entry === "number"))) {
8382
10032
  errors.push("items[" + index + "].embeddings[" + embeddingIndex + "].embedding/vector must be a number array");
8383
10033
  }
8384
10034
  if (embedding.metadata !== void 0 && !isPlainRecord(embedding.metadata)) {
@@ -8412,7 +10062,15 @@ function validateProvenanceItems(item, index, errors) {
8412
10062
  if (!isPrivacyClassification(prov.privacy)) errors.push(at + ".privacy must be one of private, sanitized, public");
8413
10063
  }
8414
10064
  }
8415
- var V2_ONLY_RECORD_FIELDS = ["search", "chunks", "embeddings", "skos_concepts", "skos_relations", "compatibility"];
10065
+ var V2_ONLY_RECORD_FIELDS = [
10066
+ "search",
10067
+ "chunks",
10068
+ "embeddings",
10069
+ "skos_concepts",
10070
+ "skos_relations",
10071
+ "provenance_events",
10072
+ "compatibility"
10073
+ ];
8416
10074
  var V2_ONLY_SOURCE_FIELDS = ["origin", "generated", "checksum", "updated_at"];
8417
10075
  var V2_ONLY_RELATIONSHIP_FIELDS = ["target_path", "direction", "metadata"];
8418
10076
  function forbidV2FieldsOnV1Record(item, index, errors) {
@@ -8444,8 +10102,9 @@ function forbidV2FieldsOnV1Record(item, index, errors) {
8444
10102
  }
8445
10103
  function validateAiwgFortemiIndexExport(value) {
8446
10104
  const errors = [];
8447
- const counts = {};
8448
- const data = value;
10105
+ const counts = /* @__PURE__ */ Object.create(null);
10106
+ const data = isPlainRecord(value) ? value : {};
10107
+ if (!isPlainRecord(value)) errors.push("index export must be an object");
8449
10108
  if (!isSupportedIndexSchemaVersion(data?.schema_version)) {
8450
10109
  errors.push("schema_version must be aiwg.fortemi.index.export.v1 or aiwg.fortemi.index.export.v2");
8451
10110
  }
@@ -8469,17 +10128,25 @@ function validateAiwgFortemiIndexExport(value) {
8469
10128
  }
8470
10129
  const ids = /* @__PURE__ */ new Set();
8471
10130
  let previousId = "";
8472
- for (const [index, item] of (data.items ?? []).entries()) {
10131
+ const items = Array.isArray(data.items) ? data.items : [];
10132
+ for (const [index, item] of items.entries()) {
10133
+ if (!isPlainRecord(item)) {
10134
+ errors.push("items[" + index + "] must be an object");
10135
+ continue;
10136
+ }
8473
10137
  for (const field of REQUIRED_RECORD_FIELDS) {
8474
10138
  if (!(field in item)) errors.push("items[" + index + "]." + field + " is required");
8475
10139
  }
8476
10140
  if (!isSupportedRecordSchemaVersion(item.schema_version)) {
8477
10141
  errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1 or aiwg.fortemi.index.record.v2");
8478
10142
  }
10143
+ if (data.schema_version === "aiwg.fortemi.index.export.v1" && item.schema_version === "aiwg.fortemi.index.record.v2") {
10144
+ errors.push("items[" + index + "].schema_version must match aiwg.fortemi.index.export.v1");
10145
+ }
8479
10146
  if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
8480
10147
  if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
8481
10148
  if (hasString(item.id)) ids.add(item.id);
8482
- if (previousId && hasString(item.id) && previousId.localeCompare(item.id) > 0) {
10149
+ if (previousId && hasString(item.id) && previousId > item.id) {
8483
10150
  errors.push("items must be sorted by id: " + previousId + " before " + item.id);
8484
10151
  }
8485
10152
  if (hasString(item.id)) previousId = item.id;
@@ -8520,7 +10187,8 @@ function assertAiwgFortemiIndexExport(value) {
8520
10187
  }
8521
10188
  function validateAiwgFortemiChunkManifest(value) {
8522
10189
  const errors = [];
8523
- const data = value;
10190
+ const data = isPlainRecord(value) ? value : {};
10191
+ if (!isPlainRecord(value)) errors.push("chunk manifest must be an object");
8524
10192
  if (data?.schema_version !== "aiwg.fortemi.index.chunk-manifest.v1") {
8525
10193
  errors.push("schema_version must be aiwg.fortemi.index.chunk-manifest.v1");
8526
10194
  }
@@ -8545,7 +10213,9 @@ function validateAiwgFortemiChunkManifest(value) {
8545
10213
  }
8546
10214
  }
8547
10215
  }
8548
- if (data.detail !== void 0) {
10216
+ if (data.detail !== void 0 && !isPlainRecord(data.detail)) {
10217
+ errors.push("detail must be an object");
10218
+ } else if (data.detail !== void 0) {
8549
10219
  if (!hasString(data.detail.href)) errors.push("detail.href is required");
8550
10220
  else if (!data.detail.href.includes("{id}")) errors.push("detail.href must contain the {id} placeholder");
8551
10221
  if (data.detail.encoding !== void 0 && data.detail.encoding !== "uri" && data.detail.encoding !== "base64url") {
@@ -8556,6 +10226,10 @@ function validateAiwgFortemiChunkManifest(value) {
8556
10226
  let expectedOffset = 0;
8557
10227
  const parts = Array.isArray(data?.parts) ? data.parts : [];
8558
10228
  for (const [index, part] of parts.entries()) {
10229
+ if (!isPlainRecord(part)) {
10230
+ errors.push("parts[" + index + "] must be an object");
10231
+ continue;
10232
+ }
8559
10233
  if (!hasString(part.href)) errors.push("parts[" + index + "].href is required");
8560
10234
  if (!hasNonNegativeInteger(part.offset)) errors.push("parts[" + index + "].offset must be a non-negative integer");
8561
10235
  if (!hasNonNegativeInteger(part.count)) errors.push("parts[" + index + "].count must be a non-negative integer");
@@ -8576,18 +10250,54 @@ function assertAiwgFortemiChunkManifest(value) {
8576
10250
  }
8577
10251
  return value;
8578
10252
  }
8579
- function validateProjectedRecords(items) {
10253
+ function validateProjectedRecordShape(item, index, errors) {
10254
+ const at = "items[" + index + "]";
10255
+ if (!item.privacy || !isPlainRecord(item.privacy)) {
10256
+ errors.push(at + "/privacy requires classification and pii");
10257
+ } else {
10258
+ if (!isPrivacyClassification(item.privacy.classification)) {
10259
+ errors.push(at + "/privacy/classification must be one of private, sanitized, public");
10260
+ }
10261
+ if (typeof item.privacy.pii !== "boolean") {
10262
+ errors.push(at + "/privacy/pii must be boolean");
10263
+ }
10264
+ }
10265
+ if (Array.isArray(item.provenance)) {
10266
+ for (const [provIndex, prov] of item.provenance.entries()) {
10267
+ if (!isPlainRecord(prov)) {
10268
+ errors.push(at + "/provenance/" + provIndex + " must be an object");
10269
+ continue;
10270
+ }
10271
+ if (!isProvenanceConfidence(prov.confidence)) {
10272
+ errors.push(at + "/provenance/" + provIndex + "/confidence must be one of source, candidate, reviewed, rejected");
10273
+ }
10274
+ if (!isPrivacyClassification(prov.privacy)) {
10275
+ errors.push(at + "/provenance/" + provIndex + "/privacy must be one of private, sanitized, public");
10276
+ }
10277
+ }
10278
+ }
10279
+ }
10280
+ function validateProjectedRecords(items, sourceSchemaVersion) {
8580
10281
  const errors = [];
8581
10282
  const ids = /* @__PURE__ */ new Set();
8582
10283
  let previousId = "";
8583
10284
  for (const [index, item] of items.entries()) {
10285
+ if (!isPlainRecord(item)) {
10286
+ errors.push("items[" + index + "] must be an object");
10287
+ continue;
10288
+ }
10289
+ validateProjectedRecordShape(item, index, errors);
10290
+ const expectedRecordVersion = sourceSchemaVersion === "aiwg.fortemi.index.export.v2" ? "aiwg.fortemi.index.record.v2" : "aiwg.fortemi.index.record.v1";
10291
+ if (item.schema_version !== expectedRecordVersion) {
10292
+ errors.push(`items[${index}].schema_version must match ${sourceSchemaVersion}`);
10293
+ }
8584
10294
  if (!isSupportedRecordSchemaVersion(item.schema_version)) {
8585
10295
  errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1 or aiwg.fortemi.index.record.v2");
8586
10296
  }
8587
10297
  if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
8588
10298
  if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
8589
10299
  if (hasString(item.id)) ids.add(item.id);
8590
- if (previousId && hasString(item.id) && previousId.localeCompare(item.id) > 0) {
10300
+ if (previousId && hasString(item.id) && previousId > item.id) {
8591
10301
  errors.push("items must be sorted by id: " + previousId + " before " + item.id);
8592
10302
  }
8593
10303
  if (hasString(item.id)) previousId = item.id;
@@ -8603,15 +10313,13 @@ function validateProjectedRecords(items) {
8603
10313
  }
8604
10314
  if (!Array.isArray(item.tags)) errors.push("items[" + index + "].tags must be an array");
8605
10315
  if (!Array.isArray(item.concepts)) errors.push("items[" + index + "].concepts must be an array");
8606
- if (!item.privacy || !hasString(item.privacy.classification)) {
8607
- errors.push("items[" + index + "].privacy.classification is required");
8608
- }
8609
10316
  }
8610
10317
  return errors;
8611
10318
  }
8612
10319
  function validateAiwgFortemiChunkPart(value, partRef, manifest) {
8613
10320
  const errors = [];
8614
- const data = value;
10321
+ const data = isPlainRecord(value) ? value : {};
10322
+ if (!isPlainRecord(value)) errors.push("chunk part must be an object");
8615
10323
  if (data?.schema_version !== "aiwg.fortemi.index.chunk.v1") {
8616
10324
  errors.push("schema_version must be aiwg.fortemi.index.chunk.v1");
8617
10325
  }
@@ -8628,12 +10336,16 @@ function validateAiwgFortemiChunkPart(value, partRef, manifest) {
8628
10336
  }
8629
10337
  if (Array.isArray(data?.items)) {
8630
10338
  if (manifest?.projection) {
8631
- errors.push(...validateProjectedRecords(data.items).map((error) => "items." + error));
10339
+ errors.push(...validateProjectedRecords(
10340
+ data.items,
10341
+ manifest.source_export_schema_version ?? "aiwg.fortemi.index.export.v1"
10342
+ ).map((error) => "items." + error));
8632
10343
  } else {
8633
10344
  const validation = validateAiwgFortemiIndexExport({
8634
- schema_version: "aiwg.fortemi.index.export.v1",
10345
+ schema_version: manifest?.source_export_schema_version ?? "aiwg.fortemi.index.export.v1",
8635
10346
  generated_at: manifest?.generated_at ?? "1970-01-01T00:00:00.000Z",
8636
10347
  source: manifest?.source ?? { repo: "chunk", privacy: "public" },
10348
+ ...(manifest?.source_export_schema_version ?? "aiwg.fortemi.index.export.v1") === "aiwg.fortemi.index.export.v2" ? { compatibility: { previous_schema_version: "aiwg.fortemi.index.export.v1", strategy: "supported" } } : {},
8637
10349
  items: data.items
8638
10350
  });
8639
10351
  errors.push(...validation.errors.map((error) => "items." + error));
@@ -8720,15 +10432,17 @@ function recordTitle(item) {
8720
10432
  }
8721
10433
  function recordText(item) {
8722
10434
  const base = item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
8723
- const extractedText = "binary_sources" in item ? item.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "" : "";
10435
+ const extractedText = binarySourceText(item);
8724
10436
  return [base, extractedText].filter(Boolean).join("\n");
8725
10437
  }
10438
+ function binarySourceText(item) {
10439
+ return "binary_sources" in item ? item.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "" : "";
10440
+ }
8726
10441
  function defaultEmbeddingInput(record, granularity) {
8727
10442
  const title = recordTitle(record);
8728
10443
  const text = recordText(record);
8729
10444
  if (granularity === "title-summary") {
8730
- const extractedText = record.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "";
8731
- return [title, record.search?.summary ?? "", extractedText].filter(Boolean).join("\n");
10445
+ return [title, record.search?.summary ?? "", binarySourceText(record)].filter(Boolean).join("\n");
8732
10446
  }
8733
10447
  return [title, text].filter(Boolean).join("\n");
8734
10448
  }
@@ -8736,6 +10450,137 @@ function generatedAtString(value) {
8736
10450
  if (value instanceof Date) return value.toISOString();
8737
10451
  return value ?? (/* @__PURE__ */ new Date()).toISOString();
8738
10452
  }
10453
+ var SHA256_K = [
10454
+ 1116352408,
10455
+ 1899447441,
10456
+ 3049323471,
10457
+ 3921009573,
10458
+ 961987163,
10459
+ 1508970993,
10460
+ 2453635748,
10461
+ 2870763221,
10462
+ 3624381080,
10463
+ 310598401,
10464
+ 607225278,
10465
+ 1426881987,
10466
+ 1925078388,
10467
+ 2162078206,
10468
+ 2614888103,
10469
+ 3248222580,
10470
+ 3835390401,
10471
+ 4022224774,
10472
+ 264347078,
10473
+ 604807628,
10474
+ 770255983,
10475
+ 1249150122,
10476
+ 1555081692,
10477
+ 1996064986,
10478
+ 2554220882,
10479
+ 2821834349,
10480
+ 2952996808,
10481
+ 3210313671,
10482
+ 3336571891,
10483
+ 3584528711,
10484
+ 113926993,
10485
+ 338241895,
10486
+ 666307205,
10487
+ 773529912,
10488
+ 1294757372,
10489
+ 1396182291,
10490
+ 1695183700,
10491
+ 1986661051,
10492
+ 2177026350,
10493
+ 2456956037,
10494
+ 2730485921,
10495
+ 2820302411,
10496
+ 3259730800,
10497
+ 3345764771,
10498
+ 3516065817,
10499
+ 3600352804,
10500
+ 4094571909,
10501
+ 275423344,
10502
+ 430227734,
10503
+ 506948616,
10504
+ 659060556,
10505
+ 883997877,
10506
+ 958139571,
10507
+ 1322822218,
10508
+ 1537002063,
10509
+ 1747873779,
10510
+ 1955562222,
10511
+ 2024104815,
10512
+ 2227730452,
10513
+ 2361852424,
10514
+ 2428436474,
10515
+ 2756734187,
10516
+ 3204031479,
10517
+ 3329325298
10518
+ ];
10519
+ function rotr(value, bits) {
10520
+ return value >>> bits | value << 32 - bits;
10521
+ }
10522
+ function sha256Hex2(data) {
10523
+ const bitLength = data.length * 8;
10524
+ const padded = new Uint8Array(data.length + 9 + 63 >> 6 << 6);
10525
+ padded.set(data);
10526
+ padded[data.length] = 128;
10527
+ const view = new DataView(padded.buffer);
10528
+ view.setUint32(padded.length - 4, bitLength >>> 0);
10529
+ view.setUint32(padded.length - 8, Math.floor(bitLength / 4294967296));
10530
+ let h0 = 1779033703;
10531
+ let h1 = 3144134277;
10532
+ let h2 = 1013904242;
10533
+ let h3 = 2773480762;
10534
+ let h4 = 1359893119;
10535
+ let h5 = 2600822924;
10536
+ let h6 = 528734635;
10537
+ let h7 = 1541459225;
10538
+ const w = new Uint32Array(64);
10539
+ for (let offset = 0; offset < padded.length; offset += 64) {
10540
+ for (let i = 0; i < 16; i++) w[i] = view.getUint32(offset + i * 4);
10541
+ for (let i = 16; i < 64; i++) {
10542
+ const s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ w[i - 15] >>> 3;
10543
+ const s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ w[i - 2] >>> 10;
10544
+ w[i] = w[i - 16] + s0 + w[i - 7] + s1 >>> 0;
10545
+ }
10546
+ let a = h0;
10547
+ let b = h1;
10548
+ let c = h2;
10549
+ let d = h3;
10550
+ let e = h4;
10551
+ let f = h5;
10552
+ let g = h6;
10553
+ let h = h7;
10554
+ for (let i = 0; i < 64; i++) {
10555
+ const s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
10556
+ const ch = e & f ^ ~e & g;
10557
+ const temp1 = h + s1 + ch + SHA256_K[i] + w[i] >>> 0;
10558
+ const s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
10559
+ const maj = a & b ^ a & c ^ b & c;
10560
+ const temp2 = s0 + maj >>> 0;
10561
+ h = g;
10562
+ g = f;
10563
+ f = e;
10564
+ e = d + temp1 >>> 0;
10565
+ d = c;
10566
+ c = b;
10567
+ b = a;
10568
+ a = temp1 + temp2 >>> 0;
10569
+ }
10570
+ h0 = h0 + a >>> 0;
10571
+ h1 = h1 + b >>> 0;
10572
+ h2 = h2 + c >>> 0;
10573
+ h3 = h3 + d >>> 0;
10574
+ h4 = h4 + e >>> 0;
10575
+ h5 = h5 + f >>> 0;
10576
+ h6 = h6 + g >>> 0;
10577
+ h7 = h7 + h >>> 0;
10578
+ }
10579
+ return [h0, h1, h2, h3, h4, h5, h6, h7].map((part) => part.toString(16).padStart(8, "0")).join("");
10580
+ }
10581
+ function computeAiwgIndexHash(data) {
10582
+ return `sha256:${sha256Hex2(data)}`;
10583
+ }
8739
10584
  async function buildAiwgStaticEmbeddingSet(index, options) {
8740
10585
  assertAiwgFortemiIndexExport(index);
8741
10586
  const granularity = options.granularity ?? "body";
@@ -8751,7 +10596,7 @@ async function buildAiwgStaticEmbeddingSet(index, options) {
8751
10596
  record_id: record.id,
8752
10597
  embedding,
8753
10598
  granularity,
8754
- input_hash: computeHash(new TextEncoder().encode(input)),
10599
+ input_hash: computeAiwgIndexHash(new TextEncoder().encode(input)),
8755
10600
  source_path: record.source.path
8756
10601
  });
8757
10602
  }
@@ -8796,6 +10641,7 @@ function recordSearchValues(item) {
8796
10641
  return values.filter((value) => typeof value === "string" && value.length > 0);
8797
10642
  }
8798
10643
  function buildAiwgChunkedIndex(index, options = {}) {
10644
+ assertAiwgFortemiIndexExport(index);
8799
10645
  const partSize = hasPositiveInteger(options.partSize) ? options.partSize : 500;
8800
10646
  const projection = options.projection;
8801
10647
  const idEncoding = options.idEncoding ?? "base64url";
@@ -8875,35 +10721,88 @@ function queryMatches(item, q) {
8875
10721
  return matches;
8876
10722
  }
8877
10723
  var DISCOVERY_STOPWORDS = /* @__PURE__ */ new Set([
10724
+ "the",
8878
10725
  "a",
8879
10726
  "an",
8880
10727
  "and",
8881
- "are",
8882
- "as",
10728
+ "or",
10729
+ "of",
8883
10730
  "for",
8884
- "from",
8885
- "how",
8886
- "i",
10731
+ "to",
8887
10732
  "in",
10733
+ "on",
10734
+ "with",
10735
+ "into",
10736
+ "from",
8888
10737
  "is",
10738
+ "are",
10739
+ "be",
10740
+ "i",
10741
+ "we",
10742
+ "my",
10743
+ "it",
10744
+ "you",
8889
10745
  "me",
8890
- "of",
8891
- "on",
8892
- "or",
10746
+ "us",
10747
+ "your",
10748
+ "our",
10749
+ "this",
10750
+ "that",
10751
+ "these",
10752
+ "those",
10753
+ "there",
10754
+ "here",
10755
+ "some",
10756
+ "any",
10757
+ "all",
10758
+ "also",
8893
10759
  "please",
8894
- "the",
8895
- "to",
8896
- "use",
8897
- "with"
10760
+ "about",
10761
+ "how",
10762
+ "what",
10763
+ "which",
10764
+ "where",
10765
+ "when",
10766
+ "who",
10767
+ "why",
10768
+ "find",
10769
+ "give",
10770
+ "show",
10771
+ "need",
10772
+ "want",
10773
+ "looking",
10774
+ "look",
10775
+ "help",
10776
+ "do",
10777
+ "does",
10778
+ "did",
10779
+ "can",
10780
+ "could",
10781
+ "should",
10782
+ "would",
10783
+ "will",
10784
+ "handle",
10785
+ "handles",
10786
+ "handling",
10787
+ "aiwg",
10788
+ "skill",
10789
+ "skills",
10790
+ "agent",
10791
+ "agents",
10792
+ "command",
10793
+ "commands",
10794
+ "rule",
10795
+ "rules",
10796
+ "flow",
10797
+ "flows",
10798
+ "workflow",
10799
+ "workflows"
8898
10800
  ]);
8899
- function normalizeDiscoveryText(value) {
8900
- return value.toLowerCase().replace(/[_/]+/g, " ").replace(/[^a-z0-9.-]+/g, " ").trim();
8901
- }
8902
- function canonicalDiscoveryName(value) {
8903
- return normalizeDiscoveryText(value).replace(/[\s.-]+/g, "");
10801
+ function normalizeDiscoveryName(value) {
10802
+ return value.toLowerCase().replace(/[-_\s]+/g, " ").trim();
8904
10803
  }
8905
10804
  function discoveryTokens(value) {
8906
- return normalizeDiscoveryText(value).split(/\s+/).filter((token) => token.length > 1 && !DISCOVERY_STOPWORDS.has(token));
10805
+ return value.toLowerCase().split(/[^a-z0-9-]+/).filter((token) => token.length > 1 && !DISCOVERY_STOPWORDS.has(token));
8907
10806
  }
8908
10807
  function facetValues(item, names) {
8909
10808
  return names.flatMap((name) => item.facets[name] ?? []);
@@ -8913,17 +10812,58 @@ function addDiscoveryMatch(matches, match) {
8913
10812
  matches.push(match);
8914
10813
  }
8915
10814
  }
8916
- function tokenOverlapScore(tokens, value) {
8917
- if (tokens.length === 0 || !value) return 0;
8918
- const normalized = normalizeDiscoveryText(value);
8919
- const hits = tokens.filter((token) => normalized.includes(token)).length;
8920
- return hits / tokens.length;
10815
+ function damerauLevenshteinAtMostOne(left, right) {
10816
+ if (left === right) return true;
10817
+ if (Math.abs(left.length - right.length) > 1) return false;
10818
+ if (left.length === right.length) {
10819
+ let firstDiff = -1;
10820
+ let diffCount = 0;
10821
+ for (let index = 0; index < left.length; index += 1) {
10822
+ if (left[index] !== right[index]) {
10823
+ if (firstDiff < 0) firstDiff = index;
10824
+ diffCount += 1;
10825
+ }
10826
+ }
10827
+ if (diffCount === 1) return true;
10828
+ return diffCount === 2 && firstDiff + 1 < left.length && left[firstDiff] === right[firstDiff + 1] && left[firstDiff + 1] === right[firstDiff];
10829
+ }
10830
+ const shorter = left.length < right.length ? left : right;
10831
+ const longer = left.length < right.length ? right : left;
10832
+ let shorterIndex = 0;
10833
+ let longerIndex = 0;
10834
+ let edits = 0;
10835
+ while (shorterIndex < shorter.length && longerIndex < longer.length) {
10836
+ if (shorter[shorterIndex] === longer[longerIndex]) {
10837
+ shorterIndex += 1;
10838
+ longerIndex += 1;
10839
+ } else {
10840
+ edits += 1;
10841
+ if (edits > 1) return false;
10842
+ longerIndex += 1;
10843
+ }
10844
+ }
10845
+ return true;
10846
+ }
10847
+ function nearDiscoveryNameMatch(query, name) {
10848
+ const queryParts = normalizeDiscoveryName(query).split(/\s+/).filter(Boolean);
10849
+ const nameParts = normalizeDiscoveryName(name).split(/\s+/).filter(Boolean);
10850
+ if (queryParts.length !== nameParts.length) return false;
10851
+ return queryParts.every((part, index) => {
10852
+ const target = nameParts[index];
10853
+ if (part === target) return true;
10854
+ if (part.length < 5 || target.length < 5) return false;
10855
+ return damerauLevenshteinAtMostOne(part, target);
10856
+ });
10857
+ }
10858
+ function lowerValues(values) {
10859
+ return values.map((value) => value.toLowerCase());
8921
10860
  }
8922
- function discoveryMatches(item, query) {
10861
+ function discoveryMatches(item, query, options = {}) {
8923
10862
  if (!query) return [];
8924
10863
  const matches = [];
8925
10864
  const tokens = discoveryTokens(query);
8926
- const canonicalQuery = canonicalDiscoveryName(query);
10865
+ const lower = tokens.length > 0 ? tokens.join(" ") : query.toLowerCase().trim();
10866
+ const rawLower = query.toLowerCase().trim();
8927
10867
  const idParts = item.id.split(/[:/]/);
8928
10868
  const names = [
8929
10869
  item.id,
@@ -8949,33 +10889,66 @@ function discoveryMatches(item, query) {
8949
10889
  ...item.tags
8950
10890
  ].filter((value) => hasString(value));
8951
10891
  const sourceValues = [item.source?.path, item.source?.repo_relative_path, item.source?.locator].filter((value) => hasString(value));
10892
+ const title = recordTitle(item);
10893
+ const summary = [item.search?.summary, recordText(item)].filter((value) => hasString(value)).join("\n");
10894
+ const type = item.search?.type ?? item.type;
10895
+ const tags = [...item.search?.tags ?? [], ...item.tags];
10896
+ const useMultiToken = tokens.length > 1;
10897
+ const minHits = useMultiToken ? options.relaxOverlap ? 1 : Math.ceil(tokens.length / 2) : 1;
10898
+ const overlapOK = (hits) => useMultiToken && hits >= minHits;
10899
+ const addInfo = (match) => addDiscoveryMatch(matches, { ...match, score: 0 });
10900
+ const addScore = (score2) => {
10901
+ if (score2 > 0) addDiscoveryMatch(matches, { field: "id", value: item.id, score: score2, reason: "aiwg discovery score" });
10902
+ };
8952
10903
  for (const name of names) {
8953
- const canonicalName = canonicalDiscoveryName(name);
8954
- if (!canonicalName) continue;
8955
- if (canonicalName === canonicalQuery) {
8956
- addDiscoveryMatch(matches, { field: "id", value: name, score: 80, reason: "exact canonical name" });
8957
- } else if (canonicalName.includes(canonicalQuery) || canonicalQuery.includes(canonicalName)) {
8958
- addDiscoveryMatch(matches, { field: "id", value: name, score: 48, reason: "near canonical name" });
10904
+ if (normalizeDiscoveryName(query) === normalizeDiscoveryName(name)) {
10905
+ addInfo({ field: "id", value: name, reason: "exact canonical name" });
10906
+ addScore(1.001);
10907
+ return matches;
10908
+ }
10909
+ if (nearDiscoveryNameMatch(query, name)) {
10910
+ addInfo({ field: "id", value: name, reason: "near canonical name" });
10911
+ addScore(0.951);
10912
+ return matches;
8959
10913
  }
8960
10914
  }
8961
- const title = recordTitle(item);
8962
- const titleOverlap = tokenOverlapScore(tokens, title);
8963
- if (titleOverlap > 0) addDiscoveryMatch(matches, { field: "title", value: title, score: 18 * titleOverlap, reason: "title token overlap" });
8964
- for (const trigger of triggers) {
8965
- const overlap = tokenOverlapScore(tokens, trigger);
8966
- if (overlap > 0) addDiscoveryMatch(matches, { field: "facet", value: trigger, score: 34 * overlap, reason: "trigger phrase" });
10915
+ let score = 0;
10916
+ const scoreText = (field, value, reason, exactScore, overlapScore, weight, exactBonus = 0) => {
10917
+ const normalized = value.toLowerCase();
10918
+ if (normalized.includes(lower)) {
10919
+ score += exactScore * weight;
10920
+ if (normalized === lower) score += exactBonus;
10921
+ addInfo({ field, value, reason });
10922
+ } else if (useMultiToken) {
10923
+ const hits = tokens.filter((token) => normalized.includes(token)).length;
10924
+ if (overlapOK(hits)) {
10925
+ score += overlapScore * weight * (hits / tokens.length);
10926
+ addInfo({ field, value, reason });
10927
+ }
10928
+ }
10929
+ };
10930
+ for (const trigger of lowerValues(triggers)) {
10931
+ if (trigger === lower || trigger === rawLower) {
10932
+ addInfo({ field: "facet", value: trigger, reason: "trigger phrase" });
10933
+ addScore(1.0008);
10934
+ return matches;
10935
+ }
8967
10936
  }
8968
- for (const capability of capabilities) {
8969
- const overlap = tokenOverlapScore(tokens, capability);
8970
- if (overlap > 0) addDiscoveryMatch(matches, { field: "concept", value: capability, score: 22 * overlap, reason: "capability overlap" });
10937
+ for (const trigger of triggers) {
10938
+ scoreText("facet", trigger, "trigger phrase", 0.25, 0.06, 4);
8971
10939
  }
8972
- const text = recordText(item);
8973
- const textOverlap = tokenOverlapScore(tokens, text);
8974
- if (textOverlap > 0) addDiscoveryMatch(matches, { field: "text", value: text, score: 8 * textOverlap, reason: "body token overlap" });
10940
+ for (const capability of capabilities) scoreText("concept", capability, "capability overlap", 0.2, 0.1, 2);
10941
+ scoreText("title", title, "title token overlap", 0.3, 0.08, 3, 0.2);
10942
+ for (const tag of tags) scoreText("tag", tag, "tag token overlap", 0.2, 0.05, 2);
10943
+ if (summary) scoreText("text", summary, "body token overlap", 0.15, 0.04, 1);
8975
10944
  for (const source of sourceValues) {
8976
- const overlap = tokenOverlapScore(tokens, source);
8977
- if (overlap > 0) addDiscoveryMatch(matches, { field: "source", value: source, score: 2 * overlap, reason: "path overlap" });
10945
+ scoreText("source", source, "path overlap", 0.1, 0.03, 1);
10946
+ }
10947
+ if (type.toLowerCase().includes(lower)) {
10948
+ score += 0.1;
10949
+ addInfo({ field: "facet", value: type, reason: "type overlap" });
8978
10950
  }
10951
+ addScore(Math.min(score, 1));
8979
10952
  return matches;
8980
10953
  }
8981
10954
  function rankMatches(matches, weights) {
@@ -9001,13 +10974,13 @@ function createSnippet(item, matches, q, maxLength) {
9001
10974
  const firstMatch = textMatch ?? titleMatch ?? matches[0];
9002
10975
  return clipSnippet(firstMatch?.value ?? recordText(item), q, maxLength);
9003
10976
  }
9004
- function createRankedEntries(items, q, options, ordinalBase = 0) {
10977
+ function createRankedEntries(items, q, options, ordinalBase = 0, discoveryOptions = {}) {
9005
10978
  const weights = { ...DEFAULT_QUERY_WEIGHTS, ...options.weights };
9006
10979
  const profile = options.searchProfile ?? "default";
9007
10980
  return items.map((item, ordinal) => ({
9008
10981
  item,
9009
10982
  ordinal: ordinalBase + ordinal,
9010
- matches: profile === "aiwg-discovery" ? discoveryMatches(item, q) : queryMatches(item, q)
10983
+ matches: profile === "aiwg-discovery" ? discoveryMatches(item, q, discoveryOptions) : queryMatches(item, q)
9011
10984
  })).filter(({ item, matches }) => {
9012
10985
  if (q && matches.length === 0) return false;
9013
10986
  if (options.types && !options.types.includes(item.type)) return false;
@@ -9057,8 +11030,7 @@ function queryAiwgFortemiIndex(index, query = "", options = {}) {
9057
11030
  const q = query.trim().toLowerCase();
9058
11031
  const entries = createRankedEntries(index.items, q, options);
9059
11032
  if (entries.length === 0 && q && options.searchProfile === "aiwg-discovery") {
9060
- const relaxed = discoveryTokens(q).join(" ");
9061
- return createQueryResultFromRankedEntries(createRankedEntries(index.items, relaxed, options), relaxed, options);
11033
+ return createQueryResultFromRankedEntries(createRankedEntries(index.items, q, options, 0, { relaxOverlap: true }), q, options);
9062
11034
  }
9063
11035
  return createQueryResultFromRankedEntries(entries, q, options);
9064
11036
  }
@@ -9079,7 +11051,8 @@ function cosineSimilarity2(left, right) {
9079
11051
  }
9080
11052
  function validateAiwgStaticEmbeddingSet(value) {
9081
11053
  const errors = [];
9082
- const data = value;
11054
+ const data = isPlainRecord(value) ? value : {};
11055
+ if (!isPlainRecord(value)) errors.push("embedding set must be an object");
9083
11056
  if (data?.schema_version !== "aiwg.fortemi.embedding.set.v1") errors.push("schema_version must be aiwg.fortemi.embedding.set.v1");
9084
11057
  if (!hasString(data?.id)) errors.push("id is required");
9085
11058
  if (!hasString(data?.model)) errors.push("model is required");
@@ -9087,7 +11060,12 @@ function validateAiwgStaticEmbeddingSet(value) {
9087
11060
  if (!hasString(data?.generated_at)) errors.push("generated_at is required");
9088
11061
  if (!hasString(data?.granularity)) errors.push("granularity is required");
9089
11062
  if (!Array.isArray(data?.embeddings)) errors.push("embeddings must be an array");
9090
- for (const [index, embedding] of (data.embeddings ?? []).entries()) {
11063
+ const embeddings = Array.isArray(data.embeddings) ? data.embeddings : [];
11064
+ for (const [index, embedding] of embeddings.entries()) {
11065
+ if (!isPlainRecord(embedding)) {
11066
+ errors.push("embeddings[" + index + "] must be an object");
11067
+ continue;
11068
+ }
9091
11069
  if (!hasString(embedding.record_id)) errors.push("embeddings[" + index + "].record_id is required");
9092
11070
  if (!hasString(embedding.input_hash)) errors.push("embeddings[" + index + "].input_hash is required");
9093
11071
  if (!Array.isArray(embedding.embedding)) errors.push("embeddings[" + index + "].embedding must be an array");
@@ -9117,7 +11095,10 @@ function queryAiwgSemanticIndex(index, embeddingSet, queryEmbedding, options = {
9117
11095
  }).filter((result) => result !== null && result.score >= (options.minScore ?? -1)).sort((left, right) => right.score - left.score || left.item.id.localeCompare(right.item.id)).slice(offset, offset + limit);
9118
11096
  }
9119
11097
  function queryAiwgHybridIndex(index, embeddingSet, query, queryEmbedding, options = {}) {
9120
- const lexical = queryAiwgFortemiIndex(index, query, { ...options, rank: true });
11098
+ const lexicalOptions = { ...options };
11099
+ delete lexicalOptions.limit;
11100
+ delete lexicalOptions.offset;
11101
+ const lexical = queryAiwgFortemiIndex(index, query, { ...lexicalOptions, rank: true });
9121
11102
  const semantic = queryAiwgSemanticIndex(index, embeddingSet, queryEmbedding, { limit: index.items.length });
9122
11103
  const lexicalWeight = options.lexicalWeight ?? 0.5;
9123
11104
  const semanticWeight = options.semanticWeight ?? 0.5;
@@ -9187,6 +11168,7 @@ function matchSetCacheKey(q, options) {
9187
11168
  concepts: options.concepts ?? null,
9188
11169
  privacy: options.privacy ?? null,
9189
11170
  rel: options.relationshipTargetId ?? null,
11171
+ searchProfile: options.searchProfile ?? "default",
9190
11172
  weights: { ...DEFAULT_QUERY_WEIGHTS, ...options.weights }
9191
11173
  });
9192
11174
  }
@@ -9275,15 +11257,21 @@ function edgeFromRelationship(sourceId, relationship) {
9275
11257
  }
9276
11258
  function relationshipMatches(edge, options = {}) {
9277
11259
  const type = relationshipTypeFilter(options);
9278
- const direction = options.direction ?? "both";
9279
11260
  if (type && edge.type !== type) return false;
9280
11261
  if (options.relationshipDirection && edge.direction !== options.relationshipDirection) return false;
9281
11262
  if (options.sourceId && edge.source_id !== options.sourceId) return false;
9282
11263
  if (options.targetId && edge.target_id !== options.targetId) return false;
9283
- if (direction === "out" && options.targetId && edge.target_id !== options.targetId) return false;
9284
- if (direction === "in" && options.sourceId && edge.source_id !== options.sourceId) return false;
11264
+ if (options.endpointId && edge.source_id !== options.endpointId && edge.target_id !== options.endpointId) return false;
9285
11265
  return true;
9286
11266
  }
11267
+ function assertRelationshipDirectionAnchor(options) {
11268
+ if (options.direction === "out" && !options.sourceId) {
11269
+ throw new Error("relationship direction 'out' requires sourceId");
11270
+ }
11271
+ if (options.direction === "in" && !options.targetId) {
11272
+ throw new Error("relationship direction 'in' requires targetId");
11273
+ }
11274
+ }
9287
11275
  function nodeSummary(item) {
9288
11276
  return { id: item.id, type: item.type, title: recordTitle(item) };
9289
11277
  }
@@ -9291,6 +11279,7 @@ function addNode(nodes, item) {
9291
11279
  if (item) nodes.set(item.id, nodeSummary(item));
9292
11280
  }
9293
11281
  function relationshipResultFromRecords(records, options = {}) {
11282
+ assertRelationshipDirectionAnchor(options);
9294
11283
  const byId = new Map(records.map((record) => [record.id, record]));
9295
11284
  const edges = [];
9296
11285
  for (const record of records) {
@@ -9300,7 +11289,8 @@ function relationshipResultFromRecords(records, options = {}) {
9300
11289
  edges.push(edge);
9301
11290
  }
9302
11291
  }
9303
- const limitedEdges = (options.limit ? edges.slice(0, options.limit) : edges).sort((left, right) => left.source_id.localeCompare(right.source_id) || left.target_id.localeCompare(right.target_id) || left.type.localeCompare(right.type));
11292
+ const sortedEdges = edges.sort((left, right) => left.source_id.localeCompare(right.source_id) || left.target_id.localeCompare(right.target_id) || left.type.localeCompare(right.type));
11293
+ const limitedEdges = options.limit ? sortedEdges.slice(0, options.limit) : sortedEdges;
9304
11294
  const nodes = /* @__PURE__ */ new Map();
9305
11295
  for (const edge of limitedEdges) {
9306
11296
  addNode(nodes, byId.get(edge.source_id));
@@ -9317,7 +11307,8 @@ function neighborQueryOptions(id, options = {}) {
9317
11307
  return {
9318
11308
  ...options,
9319
11309
  ...direction === "out" ? { sourceId: id } : {},
9320
- ...direction === "in" ? { targetId: id } : {}
11310
+ ...direction === "in" ? { targetId: id } : {},
11311
+ ...direction === "both" ? { endpointId: id } : {}
9321
11312
  };
9322
11313
  }
9323
11314
  function filterNeighborResult(id, result, options = {}) {
@@ -9626,13 +11617,14 @@ function createAiwgIndexController(initialIndex) {
9626
11617
  }
9627
11618
  function aiwgFortemiIndexToCommunityGraph(index, options = {}) {
9628
11619
  const ids = new Set(index.items.map((item) => item.id));
9629
- const relationshipWeights = options.relationshipWeights ?? {};
11620
+ const relationshipWeights = options.relationshipWeights ?? /* @__PURE__ */ Object.create(null);
9630
11621
  const edgeCounts = /* @__PURE__ */ new Map();
9631
11622
  for (const item of index.items) {
9632
11623
  for (const relationship of item.relationships) {
9633
11624
  if (!ids.has(relationship.target_id) && !options.includeDanglingRelationships) continue;
9634
11625
  const kind = relationship.type;
9635
- const baseWeight = relationshipWeights[kind] ?? 1;
11626
+ const configuredWeight = Object.prototype.hasOwnProperty.call(relationshipWeights, kind) ? relationshipWeights[kind] : void 0;
11627
+ const baseWeight = typeof configuredWeight === "number" && Number.isFinite(configuredWeight) ? configuredWeight : 1;
9636
11628
  const key = `${item.id}\0${relationship.target_id}\0${kind}`;
9637
11629
  const existing = edgeCounts.get(key);
9638
11630
  if (existing) existing.weight += baseWeight;
@@ -9668,9 +11660,726 @@ function communityIdsFor(item, options) {
9668
11660
  return [`type:${item.type}`];
9669
11661
  }
9670
11662
 
11663
+ // schemas/aiwg-fortemi-index-export.schema.json
11664
+ var aiwg_fortemi_index_export_schema_default = {
11665
+ $schema: "https://json-schema.org/draft/2020-12/schema",
11666
+ $id: "https://aiwg.io/schemas/aiwg-fortemi-index-export.json",
11667
+ title: "AIWG Fortemi Index Export",
11668
+ type: "object",
11669
+ additionalProperties: false,
11670
+ required: ["schema_version", "generated_at", "source", "items"],
11671
+ properties: {
11672
+ schema_version: {
11673
+ enum: ["aiwg.fortemi.index.export.v1", "aiwg.fortemi.index.export.v2"]
11674
+ },
11675
+ generated_at: {
11676
+ type: "string",
11677
+ format: "date-time"
11678
+ },
11679
+ source: {
11680
+ type: "object",
11681
+ additionalProperties: false,
11682
+ required: ["repo", "privacy"],
11683
+ properties: {
11684
+ repo: {
11685
+ type: "string",
11686
+ minLength: 1
11687
+ },
11688
+ privacy: {
11689
+ $ref: "#/$defs/privacy"
11690
+ },
11691
+ graph: {
11692
+ type: "string",
11693
+ minLength: 1
11694
+ }
11695
+ }
11696
+ },
11697
+ compatibility: {
11698
+ type: "object",
11699
+ additionalProperties: false,
11700
+ required: ["previous_schema_version", "strategy"],
11701
+ properties: {
11702
+ previous_schema_version: {
11703
+ const: "aiwg.fortemi.index.export.v1"
11704
+ },
11705
+ strategy: {
11706
+ const: "supported"
11707
+ }
11708
+ }
11709
+ },
11710
+ items: {
11711
+ type: "array",
11712
+ items: {
11713
+ $ref: "#/$defs/record"
11714
+ }
11715
+ }
11716
+ },
11717
+ allOf: [
11718
+ {
11719
+ if: {
11720
+ properties: {
11721
+ schema_version: {
11722
+ const: "aiwg.fortemi.index.export.v1"
11723
+ }
11724
+ }
11725
+ },
11726
+ then: {
11727
+ not: {
11728
+ required: ["compatibility"]
11729
+ },
11730
+ properties: {
11731
+ source: {
11732
+ not: {
11733
+ required: ["graph"]
11734
+ }
11735
+ },
11736
+ items: {
11737
+ items: {
11738
+ allOf: [
11739
+ {
11740
+ properties: {
11741
+ schema_version: {
11742
+ const: "aiwg.fortemi.index.record.v1"
11743
+ }
11744
+ }
11745
+ },
11746
+ {
11747
+ $ref: "#/$defs/v1RecordCompatibility"
11748
+ }
11749
+ ]
11750
+ }
11751
+ }
11752
+ }
11753
+ }
11754
+ },
11755
+ {
11756
+ if: {
11757
+ properties: {
11758
+ schema_version: {
11759
+ const: "aiwg.fortemi.index.export.v2"
11760
+ }
11761
+ }
11762
+ },
11763
+ then: {
11764
+ required: ["compatibility"],
11765
+ properties: {
11766
+ source: {
11767
+ required: ["graph"]
11768
+ },
11769
+ items: {
11770
+ items: {
11771
+ properties: {
11772
+ schema_version: {
11773
+ const: "aiwg.fortemi.index.record.v2"
11774
+ }
11775
+ }
11776
+ }
11777
+ }
11778
+ }
11779
+ }
11780
+ }
11781
+ ],
11782
+ $defs: {
11783
+ privacy: {
11784
+ enum: ["private", "sanitized", "public"]
11785
+ },
11786
+ recordType: {
11787
+ type: "string",
11788
+ minLength: 1
11789
+ },
11790
+ stringArray: {
11791
+ type: "array",
11792
+ items: {
11793
+ type: "string"
11794
+ }
11795
+ },
11796
+ numberArray: {
11797
+ type: "array",
11798
+ items: {
11799
+ type: "number"
11800
+ }
11801
+ },
11802
+ v1RecordCompatibility: {
11803
+ type: "object",
11804
+ not: {
11805
+ anyOf: [
11806
+ {
11807
+ required: ["name"]
11808
+ },
11809
+ {
11810
+ required: ["summary"]
11811
+ },
11812
+ {
11813
+ required: ["search"]
11814
+ },
11815
+ {
11816
+ required: ["chunks"]
11817
+ },
11818
+ {
11819
+ required: ["embeddings"]
11820
+ },
11821
+ {
11822
+ required: ["compatibility"]
11823
+ },
11824
+ {
11825
+ required: ["skos_concepts"]
11826
+ },
11827
+ {
11828
+ required: ["skos_relations"]
11829
+ },
11830
+ {
11831
+ required: ["provenance_events"]
11832
+ }
11833
+ ]
11834
+ },
11835
+ properties: {
11836
+ source: {
11837
+ not: {
11838
+ anyOf: [
11839
+ {
11840
+ required: ["origin"]
11841
+ },
11842
+ {
11843
+ required: ["generated"]
11844
+ },
11845
+ {
11846
+ required: ["checksum"]
11847
+ },
11848
+ {
11849
+ required: ["updated_at"]
11850
+ }
11851
+ ]
11852
+ }
11853
+ },
11854
+ relationships: {
11855
+ items: {
11856
+ not: {
11857
+ anyOf: [
11858
+ {
11859
+ required: ["target_path"]
11860
+ },
11861
+ {
11862
+ required: ["direction"]
11863
+ },
11864
+ {
11865
+ required: ["label"]
11866
+ },
11867
+ {
11868
+ required: ["confidence"]
11869
+ },
11870
+ {
11871
+ required: ["privacy"]
11872
+ },
11873
+ {
11874
+ required: ["metadata"]
11875
+ }
11876
+ ]
11877
+ }
11878
+ }
11879
+ },
11880
+ privacy: {
11881
+ not: {
11882
+ required: ["locality"]
11883
+ }
11884
+ }
11885
+ }
11886
+ },
11887
+ record: {
11888
+ type: "object",
11889
+ additionalProperties: false,
11890
+ required: [
11891
+ "schema_version",
11892
+ "id",
11893
+ "type",
11894
+ "source",
11895
+ "title",
11896
+ "text",
11897
+ "facets",
11898
+ "tags",
11899
+ "concepts",
11900
+ "relationships",
11901
+ "provenance",
11902
+ "privacy",
11903
+ "updated_at"
11904
+ ],
11905
+ properties: {
11906
+ schema_version: {
11907
+ enum: [
11908
+ "aiwg.fortemi.index.record.v1",
11909
+ "aiwg.fortemi.index.record.v2"
11910
+ ]
11911
+ },
11912
+ id: {
11913
+ type: "string",
11914
+ minLength: 1
11915
+ },
11916
+ type: {
11917
+ $ref: "#/$defs/recordType"
11918
+ },
11919
+ source: {
11920
+ type: "object",
11921
+ additionalProperties: false,
11922
+ required: ["path", "repo_relative_path", "locator"],
11923
+ properties: {
11924
+ path: {
11925
+ type: "string",
11926
+ minLength: 1
11927
+ },
11928
+ repo_relative_path: {
11929
+ type: "string",
11930
+ minLength: 1
11931
+ },
11932
+ locator: {
11933
+ type: "string",
11934
+ minLength: 1
11935
+ },
11936
+ origin: {
11937
+ type: "string",
11938
+ minLength: 1
11939
+ },
11940
+ generated: {
11941
+ type: "boolean"
11942
+ },
11943
+ checksum: {
11944
+ type: "string"
11945
+ },
11946
+ updated_at: {
11947
+ type: "string",
11948
+ format: "date-time"
11949
+ }
11950
+ }
11951
+ },
11952
+ title: {
11953
+ type: "string"
11954
+ },
11955
+ name: {
11956
+ type: "string"
11957
+ },
11958
+ summary: {
11959
+ type: "string"
11960
+ },
11961
+ text: {
11962
+ type: "string"
11963
+ },
11964
+ search: {
11965
+ type: "object",
11966
+ additionalProperties: false,
11967
+ required: [
11968
+ "title",
11969
+ "body",
11970
+ "triggers",
11971
+ "aliases",
11972
+ "tags",
11973
+ "frontmatter"
11974
+ ],
11975
+ properties: {
11976
+ title: {
11977
+ type: "string"
11978
+ },
11979
+ name: {
11980
+ type: "string"
11981
+ },
11982
+ summary: {
11983
+ type: "string"
11984
+ },
11985
+ body: {
11986
+ type: "string"
11987
+ },
11988
+ triggers: {
11989
+ $ref: "#/$defs/stringArray"
11990
+ },
11991
+ aliases: {
11992
+ $ref: "#/$defs/stringArray"
11993
+ },
11994
+ capability: {
11995
+ type: "string"
11996
+ },
11997
+ tags: {
11998
+ $ref: "#/$defs/stringArray"
11999
+ },
12000
+ phase: {
12001
+ type: "string"
12002
+ },
12003
+ type: {
12004
+ type: "string"
12005
+ },
12006
+ frontmatter: {
12007
+ type: "object"
12008
+ }
12009
+ }
12010
+ },
12011
+ facets: {
12012
+ type: "object",
12013
+ additionalProperties: {
12014
+ $ref: "#/$defs/stringArray"
12015
+ }
12016
+ },
12017
+ tags: {
12018
+ $ref: "#/$defs/stringArray"
12019
+ },
12020
+ concepts: {
12021
+ $ref: "#/$defs/stringArray"
12022
+ },
12023
+ relationships: {
12024
+ type: "array",
12025
+ items: {
12026
+ type: "object",
12027
+ additionalProperties: false,
12028
+ required: ["type", "target_id"],
12029
+ properties: {
12030
+ type: {
12031
+ type: "string",
12032
+ minLength: 1
12033
+ },
12034
+ target_id: {
12035
+ type: "string",
12036
+ minLength: 1
12037
+ },
12038
+ source_path: {
12039
+ type: "string"
12040
+ },
12041
+ target_path: {
12042
+ type: "string"
12043
+ },
12044
+ direction: {
12045
+ enum: ["upstream", "downstream", "related"]
12046
+ },
12047
+ label: {
12048
+ type: "string"
12049
+ },
12050
+ confidence: {
12051
+ type: "number"
12052
+ },
12053
+ privacy: {
12054
+ $ref: "#/$defs/privacy"
12055
+ },
12056
+ metadata: {
12057
+ type: "object"
12058
+ }
12059
+ }
12060
+ }
12061
+ },
12062
+ provenance: {
12063
+ type: "array",
12064
+ minItems: 1,
12065
+ items: {
12066
+ type: "object",
12067
+ additionalProperties: false,
12068
+ required: ["field", "source", "path", "confidence", "privacy"],
12069
+ properties: {
12070
+ field: {
12071
+ type: "string",
12072
+ minLength: 1
12073
+ },
12074
+ source: {
12075
+ type: "string",
12076
+ minLength: 1
12077
+ },
12078
+ path: {
12079
+ type: "string",
12080
+ minLength: 1
12081
+ },
12082
+ confidence: {
12083
+ enum: ["source", "candidate", "reviewed", "rejected"]
12084
+ },
12085
+ privacy: {
12086
+ $ref: "#/$defs/privacy"
12087
+ }
12088
+ }
12089
+ }
12090
+ },
12091
+ privacy: {
12092
+ type: "object",
12093
+ additionalProperties: false,
12094
+ required: ["classification", "pii"],
12095
+ properties: {
12096
+ classification: {
12097
+ $ref: "#/$defs/privacy"
12098
+ },
12099
+ pii: {
12100
+ type: "boolean"
12101
+ },
12102
+ locality: {
12103
+ enum: ["project", "framework", "external"]
12104
+ }
12105
+ }
12106
+ },
12107
+ chunks: {
12108
+ type: "array",
12109
+ items: {
12110
+ type: "object",
12111
+ additionalProperties: false,
12112
+ properties: {
12113
+ id: {
12114
+ type: "string",
12115
+ minLength: 1
12116
+ },
12117
+ text: {
12118
+ type: "string"
12119
+ },
12120
+ body: {
12121
+ type: "string"
12122
+ },
12123
+ summary: {
12124
+ type: "string"
12125
+ },
12126
+ source_path: {
12127
+ type: "string"
12128
+ },
12129
+ metadata: {
12130
+ type: "object"
12131
+ },
12132
+ checksum: {
12133
+ type: "string"
12134
+ }
12135
+ }
12136
+ }
12137
+ },
12138
+ embeddings: {
12139
+ type: "array",
12140
+ items: {
12141
+ type: "object",
12142
+ additionalProperties: false,
12143
+ properties: {
12144
+ id: {
12145
+ type: "string"
12146
+ },
12147
+ model: {
12148
+ type: "string"
12149
+ },
12150
+ embedding: {
12151
+ $ref: "#/$defs/numberArray"
12152
+ },
12153
+ vector: {
12154
+ $ref: "#/$defs/numberArray"
12155
+ },
12156
+ granularity: {
12157
+ type: "string"
12158
+ },
12159
+ source_path: {
12160
+ type: "string"
12161
+ },
12162
+ metadata: {
12163
+ type: "object"
12164
+ },
12165
+ chunk_id: {
12166
+ type: "string"
12167
+ },
12168
+ vector_ref: {
12169
+ type: "string"
12170
+ },
12171
+ input_hash: {
12172
+ type: "string"
12173
+ }
12174
+ }
12175
+ }
12176
+ },
12177
+ compatibility: {
12178
+ type: "object"
12179
+ },
12180
+ skos_concepts: {
12181
+ type: "array",
12182
+ items: {
12183
+ type: "object",
12184
+ additionalProperties: false,
12185
+ required: ["id", "prefLabel"],
12186
+ properties: {
12187
+ id: {
12188
+ type: "string",
12189
+ minLength: 1
12190
+ },
12191
+ prefLabel: {
12192
+ type: "string",
12193
+ minLength: 1
12194
+ },
12195
+ definition: {
12196
+ type: "string"
12197
+ },
12198
+ scheme: {
12199
+ type: "string"
12200
+ },
12201
+ notation: {
12202
+ type: "string"
12203
+ },
12204
+ uri: {
12205
+ type: "string"
12206
+ },
12207
+ altLabels: {
12208
+ $ref: "#/$defs/stringArray"
12209
+ },
12210
+ metadata: {
12211
+ type: "object"
12212
+ }
12213
+ }
12214
+ }
12215
+ },
12216
+ skos_relations: {
12217
+ type: "array",
12218
+ items: {
12219
+ type: "object",
12220
+ additionalProperties: false,
12221
+ required: ["type", "source_id", "target_id"],
12222
+ properties: {
12223
+ type: {
12224
+ type: "string",
12225
+ minLength: 1
12226
+ },
12227
+ source_id: {
12228
+ type: "string",
12229
+ minLength: 1
12230
+ },
12231
+ target_id: {
12232
+ type: "string",
12233
+ minLength: 1
12234
+ },
12235
+ source_path: {
12236
+ type: "string"
12237
+ },
12238
+ metadata: {
12239
+ type: "object"
12240
+ }
12241
+ }
12242
+ }
12243
+ },
12244
+ provenance_events: {
12245
+ type: "array",
12246
+ items: {
12247
+ type: "object",
12248
+ additionalProperties: false,
12249
+ required: ["activity"],
12250
+ properties: {
12251
+ id: {
12252
+ type: "string"
12253
+ },
12254
+ activity: {
12255
+ type: "string",
12256
+ minLength: 1
12257
+ },
12258
+ agent: {
12259
+ type: "string"
12260
+ },
12261
+ started_at: {
12262
+ type: "string",
12263
+ format: "date-time"
12264
+ },
12265
+ ended_at: {
12266
+ type: "string",
12267
+ format: "date-time"
12268
+ },
12269
+ source: {
12270
+ type: "string"
12271
+ },
12272
+ path: {
12273
+ type: "string"
12274
+ },
12275
+ confidence: {
12276
+ enum: ["source", "candidate", "reviewed", "rejected"]
12277
+ },
12278
+ privacy: {
12279
+ $ref: "#/$defs/privacy"
12280
+ },
12281
+ attributes: {
12282
+ type: "object"
12283
+ }
12284
+ }
12285
+ }
12286
+ },
12287
+ updated_at: {
12288
+ type: "string",
12289
+ format: "date-time"
12290
+ }
12291
+ }
12292
+ }
12293
+ }
12294
+ };
12295
+
12296
+ // src/aiwg-index-schema.ts
12297
+ var PROJECTED_FIELDS = [
12298
+ "schema_version",
12299
+ "id",
12300
+ "type",
12301
+ "title",
12302
+ "text",
12303
+ "facets",
12304
+ "tags",
12305
+ "concepts",
12306
+ "privacy"
12307
+ ];
12308
+ var ajvInstance2;
12309
+ var exportValidator;
12310
+ var projectedRecordValidator;
12311
+ function getAiwgFortemiIndexExportSchema() {
12312
+ return aiwg_fortemi_index_export_schema_default;
12313
+ }
12314
+ function getAjv2() {
12315
+ if (!ajvInstance2) {
12316
+ ajvInstance2 = new Ajv2020({
12317
+ allErrors: true,
12318
+ strict: false,
12319
+ validateFormats: false
12320
+ });
12321
+ ajvInstance2.addSchema(aiwg_fortemi_index_export_schema_default);
12322
+ }
12323
+ return ajvInstance2;
12324
+ }
12325
+ function formatErrors2(errors) {
12326
+ return (errors ?? []).map((error) => {
12327
+ const path = error.instancePath || "(root)";
12328
+ return `${path} ${error.message ?? "is invalid"}`;
12329
+ });
12330
+ }
12331
+ function getExportValidator() {
12332
+ exportValidator ??= getAjv2().getSchema(aiwg_fortemi_index_export_schema_default.$id) ?? getAjv2().compile(aiwg_fortemi_index_export_schema_default);
12333
+ return exportValidator;
12334
+ }
12335
+ function getProjectedRecordValidator() {
12336
+ if (!projectedRecordValidator) {
12337
+ const properties = Object.fromEntries(Object.keys(aiwg_fortemi_index_export_schema_default.$defs.record.properties).map((field) => [
12338
+ field,
12339
+ { $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/record/properties/${field}` }
12340
+ ]));
12341
+ projectedRecordValidator = getAjv2().compile({
12342
+ type: "object",
12343
+ required: PROJECTED_FIELDS,
12344
+ properties,
12345
+ additionalProperties: true,
12346
+ allOf: [
12347
+ {
12348
+ if: {
12349
+ properties: {
12350
+ schema_version: { const: "aiwg.fortemi.index.record.v1" }
12351
+ }
12352
+ },
12353
+ then: { $ref: `${aiwg_fortemi_index_export_schema_default.$id}#/$defs/v1RecordCompatibility` }
12354
+ }
12355
+ ]
12356
+ });
12357
+ }
12358
+ return projectedRecordValidator;
12359
+ }
12360
+ function validateAiwgFortemiIndexExportSchema(value) {
12361
+ const validate = getExportValidator();
12362
+ const schemaValue = value && typeof value === "object" && Array.isArray(value.items) ? {
12363
+ ...value,
12364
+ items: value.items.map((item) => {
12365
+ if (!item || typeof item !== "object" || Array.isArray(item)) return item;
12366
+ const schemaRecord = { ...item };
12367
+ Reflect.deleteProperty(schemaRecord, "binary_sources");
12368
+ return schemaRecord;
12369
+ })
12370
+ } : value;
12371
+ const valid = validate(schemaValue);
12372
+ return { valid, errors: formatErrors2(validate.errors) };
12373
+ }
12374
+ function validateAiwgFortemiProjectedRecordSchema(value) {
12375
+ const validate = getProjectedRecordValidator();
12376
+ const valid = validate(value);
12377
+ return { valid, errors: formatErrors2(validate.errors) };
12378
+ }
12379
+
9671
12380
  // src/index.ts
9672
- var VERSION = "2026.7.3";
12381
+ var VERSION = "2026.7.5";
9673
12382
 
9674
- export { AIWG_SCAN_REQUIRED_FIELDS, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, SHARD_FORMAT, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dumpDbSnapshot, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet, validateChecksums, verifyDbSnapshotMeta, verifySri };
12383
+ export { AIWG_SCAN_REQUIRED_FIELDS, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FORTEMI_COMPATIBILITY_PATH, FORTEMI_COMPATIBILITY_STATES, FORTEMI_REQUIRED_COMPATIBILITY_CAPABILITIES, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, SHARD_FORMAT, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, assertShardComponentRecord, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeBlobHash, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dumpDbSnapshot, embeddingConfigToShard, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchAndValidateFortemiCompatibility, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, formatFortemiCompatibilitySummary, fortemiCompatibilityUrl, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getAiwgFortemiIndexExportSchema, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getKnowledgeShardSchema, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsToShard, templateToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, urlLinkToShard, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgFortemiIndexExportSchema, validateAiwgFortemiProjectedRecordSchema, validateAiwgStaticEmbeddingSet, validateChecksums, validateFortemiCompatibilityResponse, validateShardArchive, validateShardComponentRecord, validateShardManifest, verifyDbSnapshotMeta, verifySri };
9675
12384
  //# sourceMappingURL=index.js.map
9676
12385
  //# sourceMappingURL=index.js.map