@fortemi/core 2026.7.1 → 2026.7.3

package/dist/index.js

@@ -596,6 +596,19 @@ var migration0009 = {
   `
 };
 
+// src/migrations/0010_attachment_text_metadata.ts
+var migration0010 = {
+  version: 10,
+  name: "0010_attachment_text_metadata",
+  sql: `
+    ALTER TABLE attachment
+      ADD COLUMN IF NOT EXISTS mime_type TEXT;
+
+    ALTER TABLE attachment
+      ADD COLUMN IF NOT EXISTS extracted_text TEXT;
+  `
+};
+
 // src/migrations/index.ts
 var allMigrations = [
   migration0001,
@@ -606,7 +619,8 @@ var allMigrations = [
   migration0006,
   migration0007,
   migration0008,
-  migration0009
+  migration0009,
+  migration0010
 ];
 
 // src/data-archive.ts
@@ -1119,6 +1133,16 @@ function buildNoteConditions(options, startIdx, includeDeleted = false) {
 }
 
 // src/repositories/embedding-sets-repository.ts
+var ATTACHMENT_TEXT_JOIN = `
+       LEFT JOIN (
+         SELECT note_id,
+                string_agg(extracted_text, ' ' ORDER BY position, created_at)
+                  FILTER (WHERE extracted_text IS NOT NULL AND extracted_text <> '') as extracted_text
+         FROM attachment
+         WHERE deleted_at IS NULL
+         GROUP BY note_id
+       ) ax ON ax.note_id = n.id`;
+var COMBINED_TEXT_VECTOR_SQL = `to_tsvector('english', (coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, '')))`;
 var DEFAULT_COMPATIBILITY = {
   model: "require-same",
   dimension: "require-same",
@@ -1498,7 +1522,7 @@ var EmbeddingSetsRepository = class {
       params.push(criteria.updatedBefore);
     }
     if (criteria.query?.trim()) {
-      conditions.push(`(n.tsv @@ plainto_tsquery('english', $${idx}) OR to_tsvector('english', coalesce(c.content, '')) @@ plainto_tsquery('english', $${idx}))`);
+      conditions.push(`(n.tsv @@ plainto_tsquery('english', $${idx}) OR ${COMBINED_TEXT_VECTOR_SQL} @@ plainto_tsquery('english', $${idx}))`);
       params.push(criteria.query);
     }
     const result = await this.db.query(
@@ -1506,6 +1530,7 @@ var EmbeddingSetsRepository = class {
        FROM embedding e
        JOIN note n ON n.id = e.note_id
        LEFT JOIN note_revised_current c ON c.note_id = n.id
+       ${ATTACHMENT_TEXT_JOIN}
        WHERE ${conditions.join(" AND ")}
        ORDER BY e.note_id, e.created_at DESC`,
       params
@@ -1623,6 +1648,17 @@ var EmbeddingSetsRepository = class {
 };
 
 // src/repositories/search-repository.ts
+var ATTACHMENT_TEXT_JOIN2 = `
+       LEFT JOIN (
+         SELECT note_id,
+                string_agg(extracted_text, ' ' ORDER BY position, created_at)
+                  FILTER (WHERE extracted_text IS NOT NULL AND extracted_text <> '') as extracted_text
+         FROM attachment
+         WHERE deleted_at IS NULL
+         GROUP BY note_id
+       ) ax ON ax.note_id = n.id`;
+var COMBINED_TEXT_SQL = `(coalesce(c.content, '') || ' ' || coalesce(ax.extracted_text, ''))`;
+var COMBINED_TEXT_VECTOR_SQL2 = `to_tsvector('english', ${COMBINED_TEXT_SQL})`;
 var SearchRepository = class {
   constructor(db, semanticAvailable = false) {
     this.db = db;
@@ -1707,7 +1743,7 @@ var SearchRepository = class {
     const { conditions, params, nextIdx } = buildNoteConditions(options, 2);
     conditions.unshift(
       `(n.tsv @@ ${tsqFn}('english', $1) OR
-        to_tsvector('english', coalesce(c.content, '')) @@ ${tsqFn}('english', $1))`
+        ${COMBINED_TEXT_VECTOR_SQL2} @@ ${tsqFn}('english', $1))`
     );
     let paramIdx = this.scopeToResolvedEmbeddingSet(conditions, params, nextIdx, resolvedEmbeddingSet);
     const allParams = [query, ...params];
@@ -1716,6 +1752,7 @@ var SearchRepository = class {
       `SELECT COUNT(*) as count
        FROM note n
        LEFT JOIN note_revised_current c ON c.note_id = n.id
+       ${ATTACHMENT_TEXT_JOIN2}
        WHERE ${where}`,
       allParams
     );
@@ -1724,17 +1761,18 @@ var SearchRepository = class {
     const result = await this.db.query(
       `SELECT n.id, n.title, n.created_at, n.updated_at,
               ts_rank(
-                setweight(n.tsv, 'A') || setweight(to_tsvector('english', coalesce(c.content, '')), 'B'),
+                setweight(n.tsv, 'A') || setweight(${COMBINED_TEXT_VECTOR_SQL2}, 'B'),
                 ${tsqFn}('english', $1)
               ) as rank,
               ts_headline(
                 'english',
-                coalesce(c.content, ''),
+                ${COMBINED_TEXT_SQL},
                 ${tsqFn}('english', $1),
                 'StartSel=<mark>, StopSel=</mark>, MaxWords=35, MinWords=15'
               ) as snippet
        FROM note n
        LEFT JOIN note_revised_current c ON c.note_id = n.id
+       ${ATTACHMENT_TEXT_JOIN2}
        WHERE ${where}
        ORDER BY rank DESC, n.created_at DESC
        LIMIT $${paramIdx++} OFFSET $${paramIdx++}`,
@@ -1750,6 +1788,7 @@ var SearchRepository = class {
       const idsResult = await this.db.query(
         `SELECT n.id FROM note n
          LEFT JOIN note_revised_current c ON c.note_id = n.id
+         ${ATTACHMENT_TEXT_JOIN2}
          WHERE ${where}`,
         allParams
       );
@@ -1796,10 +1835,11 @@ var SearchRepository = class {
     const result = await this.db.query(
       `SELECT n.id, n.title, n.created_at, n.updated_at,
               (e.vector <=> $${vecIdx}::vector) as distance,
-              LEFT(coalesce(c.content, ''), 200) as snippet
+              LEFT(${COMBINED_TEXT_SQL}, 200) as snippet
        FROM embedding e
        JOIN note n ON n.id = e.note_id
        LEFT JOIN note_revised_current c ON c.note_id = n.id
+       ${ATTACHMENT_TEXT_JOIN2}
        WHERE ${where}
        ORDER BY e.vector <=> $${vecIdx}::vector ASC
        LIMIT $${limIdx} OFFSET $${offIdx}`,
@@ -1840,7 +1880,7 @@ var SearchRepository = class {
     const textConditions = [
       ...textCond.conditions,
       `(n.tsv @@ ${tsqFn}('english', $1) OR
-        to_tsvector('english', coalesce(c.content, '')) @@ ${tsqFn}('english', $1))`
+        ${COMBINED_TEXT_VECTOR_SQL2} @@ ${tsqFn}('english', $1))`
     ];
     this.scopeToResolvedEmbeddingSet(textConditions, textCond.params, textCond.nextIdx, resolvedEmbeddingSet);
     const textWhere = textConditions.join(" AND ");
@@ -1848,11 +1888,12 @@ var SearchRepository = class {
     const textResult = await this.db.query(
       `SELECT n.id,
               ts_rank(
-                setweight(n.tsv, 'A') || setweight(to_tsvector('english', coalesce(c.content, '')), 'B'),
+                setweight(n.tsv, 'A') || setweight(${COMBINED_TEXT_VECTOR_SQL2}, 'B'),
                 ${tsqFn}('english', $1)
               ) as rank
        FROM note n
        LEFT JOIN note_revised_current c ON c.note_id = n.id
+       ${ATTACHMENT_TEXT_JOIN2}
        WHERE ${textWhere}
        ORDER BY rank DESC
        LIMIT 100`,
@@ -1886,9 +1927,10 @@ var SearchRepository = class {
     }
     const noteResult = await this.db.query(
       `SELECT n.id, n.title, n.created_at, n.updated_at,
-              LEFT(coalesce(c.content, ''), 200) as snippet
+              LEFT(${COMBINED_TEXT_SQL}, 200) as snippet
        FROM note n
        LEFT JOIN note_revised_current c ON c.note_id = n.id
+       ${ATTACHMENT_TEXT_JOIN2}
        WHERE n.id = ANY($1)`,
       [pageIds]
     );
@@ -1936,9 +1978,10 @@ var SearchRepository = class {
     const listParams = [...params, limit, offset];
     const result = await this.db.query(
       `SELECT n.id, n.title, n.created_at, n.updated_at,
-              LEFT(coalesce(c.content, ''), 200) as snippet
+              LEFT(${COMBINED_TEXT_SQL}, 200) as snippet
        FROM note n
        LEFT JOIN note_revised_current c ON c.note_id = n.id
+       ${ATTACHMENT_TEXT_JOIN2}
        WHERE ${where}
        ORDER BY n.created_at DESC
        LIMIT $${paramIdx++} OFFSET $${paramIdx++}`,
@@ -3811,6 +3854,29 @@ function getLlmFunction() {
   return llmFn;
 }
 
+// src/repositories/note-text.ts
+async function getNoteTextWithExtractedAttachments(db, noteId) {
+  const result = await db.query(
+    `SELECT c.content,
+            COALESCE(string_agg(a.extracted_text, E'
+' ORDER BY a.position, a.created_at)
+              FILTER (WHERE a.extracted_text IS NOT NULL AND a.extracted_text <> ''), '') as extracted_text
+       FROM note_revised_current c
+       LEFT JOIN attachment a ON a.note_id = c.note_id AND a.deleted_at IS NULL
+       WHERE c.note_id = $1
+       GROUP BY c.content`,
+    [noteId]
+  );
+  if (result.rows.length === 0) return null;
+  const content = result.rows[0].content;
+  const extractedText = result.rows[0].extracted_text ?? "";
+  return {
+    content,
+    extracted_text: extractedText,
+    combined: [content, extractedText].filter(Boolean).join("\n")
+  };
+}
+
 // src/job-queue-worker.ts
 var JOB_PRIORITIES = {
   ai_revision: 1,
@@ -4092,12 +4158,9 @@ function conceptTaggingHandler(job, db) {
   return (async () => {
     const llmFn2 = getLlmFunction();
     if (!llmFn2) return { skipped: true, reason: "no LLM function registered" };
-    const result = await db.query(
-      `SELECT content FROM note_revised_current WHERE note_id = $1`,
-      [job.note_id]
-    );
-    if (result.rows.length === 0) throw new Error(`No content found for note ${job.note_id}`);
-    const content = result.rows[0].content;
+    const noteText = await getNoteTextWithExtractedAttachments(db, job.note_id);
+    if (!noteText) throw new Error(`No content found for note ${job.note_id}`);
+    const content = noteText.combined;
     const prompt = `Task: Extract 3-5 topic tags from the text below.
 Rules:
 - Each tag is 1-3 words, lowercase
@@ -4990,9 +5053,17 @@ var AttachmentsRepository = class {
     }
     const attachmentId = generateId();
     await this.db.query(
-      `INSERT INTO attachment (id, note_id, blob_id, filename, display_name)
-       VALUES ($1, $2, $3, $4, $5)`,
-      [attachmentId, input.noteId, blobId, input.filename, input.displayName ?? null]
+      `INSERT INTO attachment (id, note_id, blob_id, filename, display_name, mime_type, extracted_text)
+       VALUES ($1, $2, $3, $4, $5, $6, $7)`,
+      [
+        attachmentId,
+        input.noteId,
+        blobId,
+        input.filename,
+        input.displayName ?? null,
+        input.mimeType ?? null,
+        input.extractedText ?? null
+      ]
     );
     return this.get(attachmentId);
   }
@@ -5055,6 +5126,7 @@ var ManageAttachmentsInputSchema = z.object({
   data_base64: z.string().optional(),
   filename: z.string().optional(),
   mime_type: z.string().optional(),
+  extracted_text: z.string().optional(),
   display_name: z.string().optional()
 });
 async function manageAttachments(db, blobStore, rawInput) {
@@ -5073,6 +5145,7 @@ async function manageAttachments(db, blobStore, rawInput) {
         data,
         filename: input.filename,
         mimeType: input.mime_type,
+        extractedText: input.extracted_text,
         displayName: input.display_name
       });
       return { action: "attach", attachment, size_bytes: data.length };
@@ -5324,12 +5397,9 @@ function averageEmbeddings(embeddings) {
 async function embeddingGenerationHandler(job, db) {
   const fn = embedFn;
   if (!fn) return { skipped: true, reason: "no embed function registered" };
-  const noteResult = await db.query(
-    `SELECT content FROM note_revised_current WHERE note_id = $1`,
-    [job.note_id]
-  );
-  if (noteResult.rows.length === 0) throw new Error(`No content for note ${job.note_id}`);
-  const content = noteResult.rows[0].content;
+  const noteText = await getNoteTextWithExtractedAttachments(db, job.note_id);
+  if (!noteText) throw new Error(`No content for note ${job.note_id}`);
+  const content = noteText.combined;
   const chunks = chunkText(content);
   const embeddings = await fn(chunks);
   const vector2 = averageEmbeddings(embeddings);
@@ -6476,8 +6546,25 @@ function packTarGz(files) {
   const tarData = encodeTar(files);
   return gzipSync(tarData);
 }
-function unpackTarGz(data) {
+var DEFAULT_MAX_DECOMPRESSED_BYTES = 256 * 1024 * 1024;
+function unpackTarGz(data, opts) {
+  const cap = opts?.maxDecompressedBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES;
+  if (data.byteLength < 18) {
+    throw new Error("Invalid gzip archive: too short");
+  }
+  const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+  const declaredSize = view.getUint32(data.byteLength - 4, true);
+  if (declaredSize > cap) {
+    throw new Error(
+      "Refusing to decompress archive: declared size " + declaredSize + " exceeds cap " + cap + " bytes"
+    );
+  }
   const tarData = gunzipSync(data);
+  if (tarData.byteLength > cap) {
+    throw new Error(
+      "Refusing to decompress archive: decompressed size " + tarData.byteLength + " exceeds cap " + cap + " bytes"
+    );
+  }
   return decodeTar(tarData);
 }
 
@@ -6512,6 +6599,7 @@ function noteToShard(note) {
     title: note.title,
     original_content: note.original_content,
     revised_content: note.revised_content,
+    ...note.binary_sources?.length ? { binary_sources: note.binary_sources } : {},
     format: note.format,
     source: note.source,
     starred: note.is_starred,
@@ -6532,6 +6620,7 @@ function noteFromShard(shard) {
     is_archived: shard.archived,
     original_content: shard.original_content,
     revised_content: shard.revised_content,
+    binary_sources: shard.binary_sources,
     tags: shard.tags,
     created_at: shard.created_at,
     updated_at: shard.updated_at,
@@ -6789,9 +6878,40 @@ async function exportShard(db, options) {
     tags.push(row.tag);
     tagsByNote.set(row.note_id, tags);
   }
+  const attachmentRows = await db.query(
+    `SELECT a.note_id,
+            a.id,
+            a.filename,
+            a.mime_type,
+            a.extracted_text,
+            b.content_hash,
+            b.size_bytes,
+            b.storage_path
+       FROM attachment a
+       JOIN attachment_blob b ON b.id = a.blob_id
+       WHERE a.deleted_at IS NULL
+       ORDER BY a.note_id, a.position, a.created_at`
+  );
+  const binarySourcesByNote = /* @__PURE__ */ new Map();
+  for (const row of attachmentRows.rows) {
+    const source = {
+      extracted_text: row.extracted_text ?? "",
+      attachment: {
+        id: row.id,
+        path: row.storage_path ?? row.filename,
+        mime: row.mime_type,
+        checksum: row.content_hash,
+        bytes: Number(row.size_bytes)
+      }
+    };
+    const sources = binarySourcesByNote.get(row.note_id) ?? [];
+    sources.push(source);
+    binarySourcesByNote.set(row.note_id, sources);
+  }
   const notes = noteRows.rows.map((row) => ({
     ...row,
-    tags: tagsByNote.get(row.id) ?? []
+    tags: tagsByNote.get(row.id) ?? [],
+    binary_sources: binarySourcesByNote.get(row.id)
   }));
   const exportedNoteIds = new Set(notes.map((n) => n.id));
   const shardNotes = notes.map((n) => noteToShard(n));
@@ -7071,6 +7191,8 @@ async function importShard(db, data, options) {
   const report = options?.onProgress;
   const warnings = [];
   const errors = [];
+  let droppedAttachmentCount = 0;
+  let notesWithDroppedAttachments = 0;
   const counts = {
     notes: 0,
     collections: 0,
@@ -7305,10 +7427,19 @@ async function importShard(db, data, options) {
             [generateId(), note.id, tag]
           );
         }
+        if (note.binary_sources?.length) {
+          droppedAttachmentCount += note.binary_sources.length;
+          notesWithDroppedAttachments++;
+        }
         counts.notes++;
         report?.({ phase: "notes", done: index + 1, total: parsedNotes.length });
         await maybeYield2(index + 1, batchSize);
       }
+      if (droppedAttachmentCount > 0) {
+        warnings.push(
+          `${droppedAttachmentCount} attachment(s) across ${notesWithDroppedAttachments} note(s) were not imported: shards currently carry attachment references (metadata + checksum) but not the binary content, so attachment bytes cannot be restored. Tracking: #237 (attachment round-trip) / server #1013 (binary contract).`
+        );
+      }
       const totalSkos = parsedSkosSchemes.length + parsedSkosConcepts.length + parsedSkosRelations.length + parsedNoteSkosTags.length;
       let doneSkos = 0;
       report?.({ phase: "skos", done: doneSkos, total: totalSkos });
@@ -7585,6 +7716,11 @@ function parseJsonArray(data) {
 
 // src/shard/shard-reader.ts
 var decoder2 = new TextDecoder();
+function assertSafeComponentName(filename) {
+  if (filename.length === 0 || filename.startsWith("/") || filename.includes("\\") || filename.includes("\0") || filename.includes(":") || filename.split("/").some((segment) => segment === "..")) {
+    throw new Error(`Refusing to read unsafe shard component path: ${JSON.stringify(filename)}`);
+  }
+}
 function parseJsonlBytes(data) {
   if (!data || data.byteLength === 0) return [];
   return decoder2.decode(data).split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
@@ -7620,6 +7756,7 @@ var UrlComponentStore = class {
     this.manifest = manifest;
   }
   async read(filename) {
+    assertSafeComponentName(filename);
     if (this.cache.has(filename)) return this.cache.get(filename);
     const response = await this.fetchImpl(`${this.baseUrl}/${filename}`);
     if (!response.ok) {
@@ -7653,7 +7790,8 @@ function tokenize(query) {
   return query.toLowerCase().split(/[^a-z0-9]+/i).filter((token) => token.length > 0);
 }
 function noteSearchText(note) {
-  return `${note.title ?? ""} ${note.original_content} ${note.revised_content ?? ""}`.toLowerCase();
+  const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
+  return `${note.title ?? ""} ${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.toLowerCase();
 }
 function countOccurrences(haystack, needle) {
   if (!needle) return 0;
@@ -7674,7 +7812,8 @@ function noteMatchesTokens(note, tokens) {
 function rankNote(note, tokens, weights) {
   if (tokens.length === 0) return 0;
   const title = (note.title ?? "").toLowerCase();
-  const content = `${note.original_content} ${note.revised_content ?? ""}`.toLowerCase();
+  const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
+  const content = `${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.toLowerCase();
   const tagText = note.tags.join(" ").toLowerCase();
   let score = 0;
   for (const token of tokens) {
@@ -7685,7 +7824,8 @@ function rankNote(note, tokens, weights) {
   return score;
 }
 function makeSnippet(note, tokens, length) {
-  const content = `${note.original_content} ${note.revised_content ?? ""}`.trim();
+  const extractedText = note.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join(" ") ?? "";
+  const content = `${note.original_content} ${note.revised_content ?? ""} ${extractedText}`.trim();
   if (tokens.length === 0) return content.slice(0, length);
   const lower = content.toLowerCase();
   const firstAt = tokens.map((token) => lower.indexOf(token)).filter((index) => index !== -1).sort((a, b) => a - b)[0] ?? 0;
@@ -8088,6 +8228,16 @@ var AIWG_SCAN_REQUIRED_FIELDS = [
   "concepts",
   "privacy"
 ];
+function isPrivacyExcluded(record, options) {
+  const privacy = record.privacy;
+  if (!privacy) return false;
+  if (privacy.classification === "private" && !options?.includePrivate) return true;
+  if (privacy.pii && !options?.includePii) return true;
+  return false;
+}
+function filterAiwgRecordsByPrivacy(records, options) {
+  return records.filter((record) => !isPrivacyExcluded(record, options));
+}
 var REQUIRED_RECORD_FIELDS = [
   "schema_version",
   "id",
@@ -8114,8 +8264,12 @@ function hasString(value) {
   return typeof value === "string" && value.length > 0;
 }
 function pushFacet(counts, name, value) {
-  counts[name] ??= {};
-  counts[name][value] = (counts[name][value] ?? 0) + 1;
+  let bucket = counts[name];
+  if (bucket === void 0) {
+    bucket = /* @__PURE__ */ Object.create(null);
+    counts[name] = bucket;
+  }
+  bucket[value] = (bucket[value] ?? 0) + 1;
 }
 function hasNonNegativeInteger(value) {
   return Number.isInteger(value) && typeof value === "number" && value >= 0;
@@ -8237,6 +8391,57 @@ function validateOptionalRichMetadata(item, index, errors) {
     errors.push("items[" + index + "].compatibility must be an object");
   }
 }
+function isPrivacyClassification(value) {
+  return value === "private" || value === "sanitized" || value === "public";
+}
+function isProvenanceConfidence(value) {
+  return value === "source" || value === "candidate" || value === "reviewed" || value === "rejected";
+}
+function validateProvenanceItems(item, index, errors) {
+  if (!Array.isArray(item.provenance)) return;
+  for (const [provIndex, prov] of item.provenance.entries()) {
+    const at = "items[" + index + "].provenance[" + provIndex + "]";
+    if (!isPlainRecord(prov)) {
+      errors.push(at + " must be an object");
+      continue;
+    }
+    if (!hasString(prov.field)) errors.push(at + ".field is required");
+    if (!hasString(prov.source)) errors.push(at + ".source is required");
+    if (!hasString(prov.path)) errors.push(at + ".path is required");
+    if (!isProvenanceConfidence(prov.confidence)) errors.push(at + ".confidence must be one of source, candidate, reviewed, rejected");
+    if (!isPrivacyClassification(prov.privacy)) errors.push(at + ".privacy must be one of private, sanitized, public");
+  }
+}
+var V2_ONLY_RECORD_FIELDS = ["search", "chunks", "embeddings", "skos_concepts", "skos_relations", "compatibility"];
+var V2_ONLY_SOURCE_FIELDS = ["origin", "generated", "checksum", "updated_at"];
+var V2_ONLY_RELATIONSHIP_FIELDS = ["target_path", "direction", "metadata"];
+function forbidV2FieldsOnV1Record(item, index, errors) {
+  if (item.schema_version !== "aiwg.fortemi.index.record.v1") return;
+  const at = "items[" + index + "]";
+  const bag = item;
+  const v2msg = " is a v2-only field and must be absent on a record.v1 record";
+  for (const field of V2_ONLY_RECORD_FIELDS) {
+    if (bag[field] !== void 0) errors.push(at + "." + field + v2msg);
+  }
+  if (isPlainRecord(item.source)) {
+    const src = item.source;
+    for (const field of V2_ONLY_SOURCE_FIELDS) {
+      if (src[field] !== void 0) errors.push(at + ".source." + field + v2msg);
+    }
+  }
+  if (isPlainRecord(item.privacy) && item.privacy.locality !== void 0) {
+    errors.push(at + ".privacy.locality" + v2msg);
+  }
+  if (Array.isArray(item.relationships)) {
+    for (const [relIndex, rel] of item.relationships.entries()) {
+      if (!isPlainRecord(rel)) continue;
+      const relBag = rel;
+      for (const field of V2_ONLY_RELATIONSHIP_FIELDS) {
+        if (relBag[field] !== void 0) errors.push(at + ".relationships[" + relIndex + "]." + field + v2msg);
+      }
+    }
+  }
+}
 function validateAiwgFortemiIndexExport(value) {
   const errors = [];
   const counts = {};
@@ -8247,10 +8452,21 @@ function validateAiwgFortemiIndexExport(value) {
   if (!hasString(data?.generated_at)) errors.push("generated_at is required");
   if (!hasString(data?.source?.repo)) errors.push("source.repo is required");
   if (!hasString(data?.source?.privacy)) errors.push("source.privacy is required");
+  else if (!isPrivacyClassification(data?.source?.privacy)) {
+    errors.push("source.privacy must be one of private, sanitized, public");
+  }
   if (!Array.isArray(data?.items)) errors.push("items must be an array");
   if (data.compatibility !== void 0 && !isPlainRecord(data.compatibility)) {
     errors.push("compatibility must be an object");
   }
+  if (data?.schema_version === "aiwg.fortemi.index.export.v1") {
+    if (isPlainRecord(data.source) && data.source.graph !== void 0) {
+      errors.push("source.graph is a v2-only field and must be absent on an export.v1 export");
+    }
+    if (data.compatibility !== void 0) {
+      errors.push("compatibility is a v2-only field and must be absent on an export.v1 export");
+    }
+  }
   const ids = /* @__PURE__ */ new Set();
   let previousId = "";
   for (const [index, item] of (data.items ?? []).entries()) {
@@ -8285,8 +8501,12 @@ function validateAiwgFortemiIndexExport(value) {
       errors.push("items[" + index + "].provenance must be a non-empty array");
     }
     validateOptionalRichMetadata(item, index, errors);
+    validateProvenanceItems(item, index, errors);
+    forbidV2FieldsOnV1Record(item, index, errors);
     if (!item.privacy || typeof item.privacy.pii !== "boolean" || !hasString(item.privacy.classification)) {
       errors.push("items[" + index + "].privacy requires classification and pii");
+    } else if (!isPrivacyClassification(item.privacy.classification)) {
+      errors.push("items[" + index + "].privacy.classification must be one of private, sanitized, public");
     }
   }
   return { valid: errors.length === 0, errors, counts };
@@ -8307,6 +8527,9 @@ function validateAiwgFortemiChunkManifest(value) {
   if (!hasString(data?.generated_at)) errors.push("generated_at is required");
   if (!hasString(data?.source?.repo)) errors.push("source.repo is required");
   if (!hasString(data?.source?.privacy)) errors.push("source.privacy is required");
+  if (data?.source_export_schema_version !== void 0 && !isSupportedIndexSchemaVersion(data.source_export_schema_version)) {
+    errors.push("source_export_schema_version must be aiwg.fortemi.index.export.v1 or aiwg.fortemi.index.export.v2 when present");
+  }
   if (!hasNonNegativeInteger(data?.total)) errors.push("total must be a non-negative integer");
   if (!hasPositiveInteger(data?.part_size)) errors.push("part_size must be a positive integer");
   if (data.facets !== void 0 && !isFacetCounts(data.facets)) {
@@ -8425,9 +8648,35 @@ function assertAiwgFortemiChunkPart(value, partRef, manifest) {
   }
   return value;
 }
+var ALLOWED_AIWG_FETCH_SCHEMES = /* @__PURE__ */ new Set(["http:", "https:", "blob:", "data:"]);
+function tryParseUrl(value) {
+  try {
+    return new URL(value);
+  } catch {
+    return null;
+  }
+}
+function resolveAiwgFetchUrl(href, baseUrl) {
+  if (baseUrl === void 0) {
+    const absolute = tryParseUrl(href);
+    if (absolute && !ALLOWED_AIWG_FETCH_SCHEMES.has(absolute.protocol)) {
+      throw new Error("Refusing AIWG index fetch with disallowed scheme: " + absolute.protocol);
+    }
+    return href;
+  }
+  const base = new URL(baseUrl);
+  const resolved = new URL(href, base);
+  if (!ALLOWED_AIWG_FETCH_SCHEMES.has(resolved.protocol)) {
+    throw new Error("Refusing AIWG index fetch with disallowed scheme: " + resolved.protocol);
+  }
+  if (resolved.origin !== base.origin) {
+    throw new Error("Refusing cross-origin AIWG index fetch: " + resolved.origin + " != " + base.origin);
+  }
+  return resolved.toString();
+}
 function createAiwgFetchChunkLoader(baseUrl) {
   return async (part) => {
-    const href = baseUrl ? new URL(part.href, baseUrl).toString() : part.href;
+    const href = resolveAiwgFetchUrl(part.href, baseUrl);
     const response = await fetch(href);
     if (!response.ok) throw new Error("Failed to fetch AIWG index chunk " + href + ": " + response.status);
     return response.json();
@@ -8447,14 +8696,14 @@ function createAiwgFetchDetailLoader(baseUrl) {
   return async (id, manifest) => {
     if (!manifest.detail?.href) throw new Error("Manifest has no detail.href for record resolution");
     const relative = aiwgDetailHrefForId(manifest.detail, id);
-    const href = baseUrl ? new URL(relative, baseUrl).toString() : relative;
+    const href = resolveAiwgFetchUrl(relative, baseUrl);
     const response = await fetch(href);
     if (!response.ok) throw new Error("Failed to fetch AIWG index detail " + href + ": " + response.status);
     return response.json();
   };
 }
 function getAiwgFortemiFacets(items) {
-  const result = {};
+  const result = /* @__PURE__ */ Object.create(null);
   for (const item of items) {
     pushFacet(result, "type", item.type);
     pushFacet(result, "privacy", item.privacy.classification);
@@ -8470,7 +8719,55 @@ function recordTitle(item) {
   return item.title ?? item.search?.title ?? item.search?.name ?? item.id;
 }
 function recordText(item) {
-  return item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
+  const base = item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
+  const extractedText = "binary_sources" in item ? item.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "" : "";
+  return [base, extractedText].filter(Boolean).join("\n");
+}
+function defaultEmbeddingInput(record, granularity) {
+  const title = recordTitle(record);
+  const text = recordText(record);
+  if (granularity === "title-summary") {
+    const extractedText = record.binary_sources?.map((source) => source.extracted_text).filter(Boolean).join("\n") ?? "";
+    return [title, record.search?.summary ?? "", extractedText].filter(Boolean).join("\n");
+  }
+  return [title, text].filter(Boolean).join("\n");
+}
+function generatedAtString(value) {
+  if (value instanceof Date) return value.toISOString();
+  return value ?? (/* @__PURE__ */ new Date()).toISOString();
+}
+async function buildAiwgStaticEmbeddingSet(index, options) {
+  assertAiwgFortemiIndexExport(index);
+  const granularity = options.granularity ?? "body";
+  const records = filterAiwgRecordsByPrivacy(options.records ?? index.items, options.privacy);
+  const embeddings = [];
+  for (const record of records) {
+    const input = options.textForRecord?.(record) ?? defaultEmbeddingInput(record, granularity);
+    const embedding = await options.backend.embed(input, record);
+    if (embedding.length !== options.backend.dimensions) {
+      throw new Error(`Embedding for ${record.id} has ${embedding.length} dimensions; expected ${options.backend.dimensions}`);
+    }
+    embeddings.push({
+      record_id: record.id,
+      embedding,
+      granularity,
+      input_hash: computeHash(new TextEncoder().encode(input)),
+      source_path: record.source.path
+    });
+  }
+  const embeddingSet = {
+    schema_version: "aiwg.fortemi.embedding.set.v1",
+    id: options.id,
+    model: options.backend.model,
+    dimensions: options.backend.dimensions,
+    generated_at: generatedAtString(options.generatedAt),
+    granularity,
+    ...options.metric ? { metric: options.metric } : {},
+    input_hash_algorithm: "sha256",
+    embeddings
+  };
+  assertAiwgStaticEmbeddingSet(embeddingSet);
+  return embeddingSet;
 }
 function recordSearchValues(item) {
   const search = item.search;
@@ -8503,7 +8800,7 @@ function buildAiwgChunkedIndex(index, options = {}) {
   const projection = options.projection;
   const idEncoding = options.idEncoding ?? "base64url";
   const detailHref = options.detailHref ?? "detail/{id}.json";
-  const items = index.items;
+  const items = filterAiwgRecordsByPrivacy(index.items, options.privacy);
   const pad = (value) => String(value).padStart(4, "0");
   const project = (record) => {
     if (!projection) return record;
@@ -8531,6 +8828,7 @@ function buildAiwgChunkedIndex(index, options = {}) {
     schema_version: "aiwg.fortemi.index.chunk-manifest.v1",
     generated_at: options.generatedAt ?? index.generated_at,
     source: index.source,
+    source_export_schema_version: index.schema_version,
     total: items.length,
     part_size: partSize,
     facets: getAiwgFortemiFacets(items),
@@ -8841,8 +9139,15 @@ function queryAiwgHybridIndex(index, embeddingSet, query, queryEmbedding, option
     };
   }).filter((result) => result !== null && result.score >= (options.minScore ?? -1)).sort((left, right) => right.score - left.score || left.item.id.localeCompare(right.item.id)).slice(offset, offset + limit);
 }
-function findAiwgStaticDuplicatePairs(index, embeddingSet, threshold = 0.9) {
+var DEFAULT_AIWG_DUPLICATE_SCAN_MAX_EMBEDDINGS = 5e3;
+function findAiwgStaticDuplicatePairs(index, embeddingSet, threshold = 0.9, options) {
   assertAiwgStaticEmbeddingSet(embeddingSet);
+  const maxEmbeddings = options?.maxEmbeddings ?? DEFAULT_AIWG_DUPLICATE_SCAN_MAX_EMBEDDINGS;
+  if (embeddingSet.embeddings.length > maxEmbeddings) {
+    throw new Error(
+      "Embedding set too large for duplicate scan: " + embeddingSet.embeddings.length + " > " + maxEmbeddings + " (raise options.maxEmbeddings to override for trusted input)"
+    );
+  }
   const byId = new Map(index.items.map((item) => [item.id, item]));
   const pairs = [];
   for (let leftIndex = 0; leftIndex < embeddingSet.embeddings.length; leftIndex += 1) {
@@ -9307,7 +9612,7 @@ function createAiwgIndexController(initialIndex) {
       notify();
     },
     createReviewDecisionExport(generatedAt) {
-      const source = index ?? (chunked ? { schema_version: "aiwg.fortemi.index.export.v1" } : null);
+      const source = index ?? (chunked ? { schema_version: chunked.manifest.source_export_schema_version ?? "aiwg.fortemi.index.export.v1" } : null);
       if (!source) throw new Error("No AIWG index export or chunked manifest loaded");
       return createAiwgReviewDecisionExport(source, reviewDecisions, generatedAt);
     },
@@ -9364,8 +9669,8 @@ function communityIdsFor(item, options) {
 }
 
 // src/index.ts
-var VERSION = "2026.7.1";
+var VERSION = "2026.7.3";
 
-export { AIWG_SCAN_REQUIRED_FIELDS, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, SHARD_FORMAT, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dumpDbSnapshot, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, findAiwgStaticDuplicatePairs, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet, validateChecksums, verifyDbSnapshotMeta, verifySri };
+export { AIWG_SCAN_REQUIRED_FIELDS, ArchiveManager, AttachmentsRepository, CURRENT_MIGRATION_HEAD, CURRENT_SHARD_VERSION, CapabilityManager, CaptureKnowledgeInputSchema, CollectionsRepository, CommunitiesRepository, DB_SNAPSHOT_SCHEMA_VERSION, DbSnapshotVersionError, EMBED_REQUEST_KIND, EMBED_RESPONSE_KIND, EmbeddingSetsRepository, FallbackRouter, FortemiToolManifest, GetNoteInputSchema, GraphRepository, JOB_CAPABILITIES, JOB_PRIORITIES, JobQueueWorker, LOCAL_ENDPOINTS, LinksRepository, ListNotesInputSchema, ManageArchiveInputSchema, ManageAttachmentsInputSchema, ManageCapabilitiesInputSchema, ManageCollectionsInputSchema, ManageLinksInputSchema, ManageNoteInputSchema, ManageTagsInputSchema, MemoryBlobStore, MigrationRunner, NotesRepository, OpenAICompatibleProvider, PGliteStorageBackend, PGliteStorageBackendFactory, PGliteWorkerClient, PGliteWorkerStorageBackend, PGliteWorkerStorageBackendFactory, ProvenanceRepository, ProviderRegistry, SHARD_FORMAT, SUPPORTED_PGLITE_VERSION, SearchInputSchema, SearchRepository, SkosRepository, TagsRepository, TransactionProxy, TypedEventBus, VERSION, aiRevisionHandler, aiwgFortemiIndexToCommunityGraph, allMigrations, appendPluginScript, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, buildAiwgStaticEmbeddingSet, buildNoteConditions, buildPluginCsp, captureKnowledge, chunkText, classifyError, classifyModel, clearPrefetchedShard, collectionFromShard, collectionToShard, computeHash, computeSri, conceptTaggingHandler, cosineSimilarity, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, createBlobStore, createCosineSemanticProvider, createCspReportHandler, createFortemi, createLegacyProvider, createPGliteBackend, createPGliteInstance, createRemoteBackend, createRoutes, createShardBackend, createWorkerEmbedFunction, defaultStorageBackendFactory, detectCommunities, detectGpuCapabilities, detectInferenceCapabilities, discoverLocalProviders, dumpDbSnapshot, embeddingFromShard, embeddingGenerationHandler, embeddingSetFromShard, embeddingSetMemberToShard, embeddingSetToShard, embeddingToShard, enqueueFullWorkflow, enqueueJob, enqueueNoteCreationJobs, estimateModelFit, estimateVramMB, estimateVramTier, exportShard, fetchPluginScript, filterAiwgRecordsByPrivacy, findAiwgStaticDuplicatePairs, fortemiManifest, fromPrefetched, generateId, getAiwgFortemiFacets, getEmbedFunction, getFortemiBridge, getFortemiSecretStore, getJobQueueStatus, getLlmFunction, getNote, getPrefetchedSha256, handleEmbedRequests, hasFortemiSecureSecrets, importShard, isPluginScriptAllowed, isShardPrefetched, linkFromShard, linkToShard, linkingHandler, listNotes, manageArchive, manageAttachments, manageCapabilities, manageCollections, manageLinks, manageNote, manageTags, matchRoute, noteFromShard, noteSkosTagToShard, noteToShard, openShard, packTarGz, parseCspReport, prefetchShard, provenanceEdgeToShard, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, registerLlmCapability, registerSemanticCapability, registerSemanticCapabilityWorker, registerServiceWorker, restoreDbSnapshot, searchTool, selectBackend, selectLlmModel, setEmbedFunction, setLlmFunction, sha256Hex, skosConceptToShard, skosRelationToShard, skosSchemeToShard, suggestTags, tagsFromShard, tagsToShard, titleGenerationHandler, unpackTarGz, unregisterLlmCapability, unregisterSemanticCapability, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet, validateChecksums, verifyDbSnapshotMeta, verifySri };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map