@askdb/rag 0.2.0-beta.3 → 0.2.0-beta.30

package/README.md

@@ -7,21 +7,25 @@ Deterministic retrieval over AskDB Schema v2 artifacts. `@askdb/rag` chunks the
 ## Install
 
 ```bash
-pnpm add @askdb/rag @askdb/core
+pnpm add @askdb/rag @askdb/core ai
 # only if you use the pgvector adapter:
 pnpm add pg
-# only if you use the OpenAI embedder helper or CLI `--embedder openai`:
-pnpm add ai @ai-sdk/openai
+# only if you use the OpenAI embedder helper:
+pnpm add @ai-sdk/openai
 ```
 
-`pg`, `ai`, and `@ai-sdk/openai` are optional peer dependencies. The chunker, in-memory store, and file store do not require them.
+`ai` is a required peer dependency of `@askdb/core`. For `@askdb/rag` itself, `pg`, `ai`, and `@ai-sdk/openai` are optional peer dependencies — the chunker, in-memory store, and file store do not require them.
 
 ## Quickstart
 
 ```ts
 import { ask, loadSchema } from "@askdb/core";
-import { buildSchemaIndex, loadChunkerSourcesFromDir } from "@askdb/rag";
-import { createMemoryStore } from "@askdb/rag/stores/memory";
+import {
+  buildSchemaIndex,
+  loadChunkerSourcesFromDir,
+  createMemoryStore,
+  createAiSdkEmbedder,
+} from "@askdb/rag";
 
 const schemaDir = "./fixtures/schemas/orders-users.schema";
 const schema = loadSchema(schemaDir);
@@ -48,24 +52,62 @@ const { sql } = await ask({
 
 ## CLI
 
+`@askdb/rag` is a library. The command line for it is `askdb rag` in the [`askdb`](https://www.npmjs.com/package/askdb) package, which reads `rag.*` and `ai.embedding` from `askdb.config.*`:
+
 ```bash
-askdb-rag index fixtures/schemas/orders-users.schema --store file
-askdb-rag query fixtures/schemas/orders-users.schema \
+npx askdb rag index fixtures/schemas/orders-users.schema --store file
+npx askdb rag query fixtures/schemas/orders-users.schema \
   --question "How much revenue did we make last month?"
 ```
 
-The default CLI embedder is a deterministic mock for smoke tests. Use `--embedder openai` with `OPENAI_API_KEY` for real embeddings.
+See the [CLI reference](https://askdb.tools/reference/cli/#askdb-rag) for every flag. This package no longer ships an `askdb-rag` binary.
 
 ## Public Surface
 
+All exports are available from the root `@askdb/rag` import. Sub-path imports are also supported and point to the same modules — use whichever style you prefer.
+
+### Core
+
 - `chunkSchema`, `chunkSchemaDir`, `chunkSchemaBundle` — deterministic Schema v2 chunking.
 - `buildSchemaIndex` — chunk, embed, upsert, write `schema.lock.json`, and return a retriever.
 - `createRetriever` — bind an existing store and embedder without indexing.
-- `@askdb/rag/stores/memory` — in-memory cosine store.
-- `@askdb/rag/stores/file` — binary embedding file plus JSON metadata.
-- `@askdb/rag/stores/pgvector` — pgvector adapter with documented setup SQL.
-- `@askdb/rag/embedders/ai-sdk` — generic AI SDK `EmbeddingModel<string>` adapter.
-- `@askdb/rag/embedders/openai` — optional OpenAI convenience helper using `text-embedding-3-small` by default.
+
+### Stores
+
+| Root import | Sub-path import | Description |
+|---|---|---|
+| `createMemoryStore` | `@askdb/rag/stores/memory` | In-memory cosine store. Zero deps. |
+| `createFileStore` | `@askdb/rag/stores/file` | Binary embedding file + JSON metadata. |
+| `createPgvectorStore` | `@askdb/rag/stores/pgvector` | pgvector adapter with documented setup SQL. Requires `pg`. Needs `dimensions` only to create its table; `ensureSchema()` refuses an existing table of another width with a `PgvectorDimensionMismatchError`. |
+
+### Embedders
+
+| Root import | Sub-path import | Description |
+|---|---|---|
+| `createAiSdkEmbedder` | `@askdb/rag/embedders/ai-sdk` | Generic AI SDK `EmbeddingModel` adapter. Requires `ai`. |
+| `detectEmbeddingDimensions` | — | Learns an embedder's vector width by embedding one short text. Use it for a new pgvector table instead of hard-coding a width. |
+| `aiSdkEmbedderId` | `@askdb/rag/embedders/ai-sdk` | The `embedderId` Studio and `askdb rag` record for an `ai.embedding` model (`ai-sdk:<provider>:<model>:<dimensions or default>`), so an index built by one is accepted by the other. |
+| `createOpenAiEmbedder` | `@askdb/rag/embedders/openai` | **Deprecated.** OpenAI convenience helper. Use `createAiSdkEmbedder` with an AI SDK embedding model (e.g. from `@ai-sdk/openai`) or one built by the `@askdb/ai` registry instead. Removed in 1.0. |
+
+### Import examples
+
+```ts
+// Everything from the root — simplest DX
+import {
+  buildSchemaIndex,
+  createMemoryStore,
+  createAiSdkEmbedder,
+  createPgvectorStore,
+} from "@askdb/rag";
+
+// Sub-path imports — same modules, explicit namespacing
+import { createMemoryStore } from "@askdb/rag/stores/memory";
+import { createFileStore } from "@askdb/rag/stores/file";
+import { createPgvectorStore } from "@askdb/rag/stores/pgvector";
+import { createAiSdkEmbedder } from "@askdb/rag/embedders/ai-sdk";
+// Recommended: bring your own embedding model (e.g. @ai-sdk/openai, or @askdb/ai's registry) + createAiSdkEmbedder
+// import { createOpenAiEmbedder } from "@askdb/rag/embedders/openai"; // deprecated, removed in 1.0
+```
 
 ## Sensitive Fields
 
@@ -73,4 +115,4 @@ Sensitive describable-layer content and sensitive identifiers are excluded from
 
 ## License
 
-Apache-2.0. See [LICENSE](./LICENSE) and [NOTICE](./NOTICE).
+Apache-2.0 © [Yahya Gilany](https://yahyagilany.io). See [LICENSE](./LICENSE) and [NOTICE](./NOTICE).

package/dist/chunker/chunker.d.ts

@@ -11,7 +11,11 @@ import type { ChunkerSources } from "./sources.js";
 export type ChunkStats = {
     totalChunks: number;
     byType: Record<Chunk["type"], number>;
-    /** Describable-layer chunks dropped because the source content references a sensitive column/table. */
+    /**
+     * Chunks dropped, plus chunks emitted with part of their describable text
+     * (a description, alias, primary entity, enum values, or note) dropped,
+     * because the content belongs to or mentions a sensitive column/table.
+     */
     sensitiveExcluded: number;
     /** Describable-layer chunks emitted **only because** `includeSensitiveDescribable: true` was set. */
     sensitiveIncluded: number;

package/dist/chunker/chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/chunker/chunker.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;;;;;GAMG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IACtC,uGAAuG;IACvG,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qGAAqG;IACrG,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,qCAAqC;AACrC,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,KAAK,EAAE,UAAU,CAAC;CACnB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,cAAc,EACvB,OAAO,GAAE,YAAiB,GACzB,WAAW,CAmMb"}
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/chunker/chunker.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAGnD;;;;;;GAMG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IACtC;;;;OAIG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qGAAqG;IACrG,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,qCAAqC;AACrC,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,KAAK,EAAE,UAAU,CAAC;CACnB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,cAAc,EACvB,OAAO,GAAE,YAAiB,GACzB,WAAW,CAqQb"}

package/dist/chunker/chunker.js

@@ -1,4 +1,6 @@
+import { createMentionMatcher, sensitiveMentionNames } from "@askdb/core";
 import { DEFAULT_CHUNK_MAX_CHARS, } from "./options.js";
+import { chunkId } from "./ids.js";
 /**
  * Deterministic chunker for a Schema v2 artifact.
  *
@@ -24,24 +26,40 @@ export function chunkSchema(sources, options = {}) {
             question: 0,
             concept: 0,
             relationship: 0,
+            "tenant-policy": 0,
         },
         sensitiveExcluded: 0,
         sensitiveIncluded: 0,
     };
     const { schema, tables: tableMarkdowns, concepts } = sources;
+    // Names whose mention marks describable text as sensitive, schema-wide
+    // (ADR 0017): every table's text, concept, and tenant policy section is
+    // checked against them.
+    const sensitiveMentions = createMentionMatcher(sensitiveMentionNames(schema));
     for (const table of schema.tables) {
+        if (table.tracked === false) {
+            continue;
+        }
         const md = tableMarkdowns[table.id];
-        const sensitiveColumnNames = table.columns
-            .filter((c) => c.sensitive)
-            .map((c) => c.name);
         if (table.sensitive && !includeSensitive) {
             stats.sensitiveExcluded++;
         }
         else {
-            chunks.push(buildTableChunk(table, schema.schemaId, includeSensitive, schema));
+            const filter = describableFilter(sensitiveMentions, includeSensitive);
+            const tableChunk = buildTableChunk(table, schema.schemaId, includeSensitive, schema, filter);
+            // A non-sensitive table whose description/aliases/column headlines
+            // mention a sensitive column is only embedded verbatim in opt-in mode.
+            if (filter.dropped)
+                stats.sensitiveExcluded++;
+            // Opt-in: a sensitive table's chunk is emitted only because of it.
+            if (table.sensitive || filter.included)
+                stats.sensitiveIncluded++;
+            chunks.push(tableChunk);
         }
         // Column chunks — one per column. Identifier + type always; describable
-        // fields only when not sensitive (or when opted in).
+        // fields only when not sensitive (or when opted in). A non-sensitive
+        // column whose describable fields name a sensitive column is treated
+        // like a sensitive column's describable layer.
         for (const col of table.columns) {
             const colNote = readColumnNote(md, col.name);
             const colSensitive = col.sensitive || table.sensitive;
@@ -49,27 +67,46 @@ export function chunkSchema(sources, options = {}) {
                 stats.sensitiveExcluded++;
                 continue;
             }
-            else if (colSensitive && includeSensitive) {
-                if (col.description || (col.aliases && col.aliases.length) || (col.enum && col.enum.length) || colNote) {
-                    stats.sensitiveIncluded++;
-                }
+            const describable = columnDescribableTexts(col, colNote);
+            if (colSensitive) {
+                // Opt-in mode: the default leaves this chunk out entirely.
+                stats.sensitiveIncluded++;
+                chunks.push(buildColumnChunk(table, col, schema.schemaId, colNote, true, true));
+                continue;
             }
-            chunks.push(buildColumnChunk(table, col, schema.schemaId, colNote, includeSensitive));
+            const describableMentionsSensitive = describable.some((text) => sensitiveMentions.mentionsAny(text));
+            if (describableMentionsSensitive && !includeSensitive) {
+                // Keep the identifier + type; drop the whole describable layer.
+                stats.sensitiveExcluded++;
+                chunks.push(buildColumnChunk(table, col, schema.schemaId, colNote, false, false));
+                continue;
+            }
+            if (describableMentionsSensitive)
+                stats.sensitiveIncluded++;
+            chunks.push(buildColumnChunk(table, col, schema.schemaId, colNote, true, describableMentionsSensitive));
         }
         // CQL chunk — excluded entirely if the body mentions a sensitive column by name.
         if (table.commonQueryLanguage) {
-            const mentionsSensitive = mentionsAnyName(table.commonQueryLanguage, sensitiveColumnNames);
+            const mentionsSensitive = sensitiveMentions.mentionsAny(table.commonQueryLanguage);
             const tableLevelSensitive = table.sensitive;
             const skip = (tableLevelSensitive || mentionsSensitive) && !includeSensitive;
+            // Counted per chunk: a long body splits into several.
+            const parts = splitLong(table.commonQueryLanguage, maxChars);
             if (skip) {
-                stats.sensitiveExcluded++;
+                stats.sensitiveExcluded += parts.length;
             }
             else {
-                if (tableLevelSensitive || mentionsSensitive) {
-                    stats.sensitiveIncluded++;
-                }
-                for (const part of splitLong(table.commonQueryLanguage, maxChars)) {
-                    chunks.push(buildCqlChunk(table, part.text, part.suffix, schema.schemaId, tableLevelSensitive || mentionsSensitive));
+                // The heading repeats the table's aliases; one naming a sensitive
+                // column is dropped like it is from the table chunk.
+                const filter = describableFilter(sensitiveMentions, includeSensitive);
+                const aliases = (table.aliases ?? []).filter((a) => filter.allow(a));
+                const sensitive = tableLevelSensitive || mentionsSensitive || filter.included;
+                if (filter.dropped)
+                    stats.sensitiveExcluded += parts.length;
+                if (sensitive)
+                    stats.sensitiveIncluded += parts.length;
+                for (const part of parts) {
+                    chunks.push(buildCqlChunk(table, aliases, part.text, part.suffix, schema.schemaId, sensitive));
                 }
             }
         }
@@ -77,8 +114,8 @@ export function chunkSchema(sources, options = {}) {
         if (md) {
             const questions = extractExampleQuestions(md);
             const businessContext = md.sections["Business context"]?.trim() ?? "";
-            const questionsMentionSensitive = questions.some((q) => mentionsAnyName(q, sensitiveColumnNames));
-            const bizMentionsSensitive = mentionsAnyName(businessContext, sensitiveColumnNames);
+            const questionsMentionSensitive = questions.some((q) => sensitiveMentions.mentionsAny(q));
+            const bizMentionsSensitive = sensitiveMentions.mentionsAny(businessContext);
             // Per-table sensitive gate: if table is sensitive OR mentions exist, skip
             // the *whole* describable layer for that table by default. Authoring rule
             // in the contract notes the chunker excludes whole chunks rather than
@@ -89,25 +126,34 @@ export function chunkSchema(sources, options = {}) {
                     stats.sensitiveExcluded += questions.length;
                 }
                 else {
-                    if (table.sensitive || questionsMentionSensitive) {
+                    // Every question heading repeats the primary entity; one naming a
+                    // sensitive column is dropped like it is from the table chunk.
+                    const filter = describableFilter(sensitiveMentions, includeSensitive);
+                    const primaryEntity = filter.allow(table.primaryEntity) ? table.primaryEntity : undefined;
+                    if (filter.dropped)
+                        stats.sensitiveExcluded += questions.length;
+                    // The questions are gated together, so in opt-in mode one naming a
+                    // sensitive column makes every question chunk of the table sensitive.
+                    const questionsSensitive = table.sensitive || questionsMentionSensitive || filter.included;
+                    if (questionsSensitive)
                         stats.sensitiveIncluded += questions.length;
-                    }
                     questions.forEach((q, i) => {
-                        chunks.push(buildQuestionChunk(table, q, i + 1, schema.schemaId, table.sensitive || mentionsAnyName(q, sensitiveColumnNames)));
+                        chunks.push(buildQuestionChunk(table, primaryEntity, q, i + 1, schema.schemaId, questionsSensitive));
                     });
                 }
             }
             // Business-context chunks are part of the describable layer; same gate.
             if (businessContext) {
                 const skipBiz = (table.sensitive || bizMentionsSensitive) && !includeSensitive;
+                const parts = splitLong(businessContext, maxChars);
                 if (skipBiz) {
-                    stats.sensitiveExcluded++;
+                    stats.sensitiveExcluded += parts.length;
                 }
                 else {
                     if (table.sensitive || bizMentionsSensitive) {
-                        stats.sensitiveIncluded++;
+                        stats.sensitiveIncluded += parts.length;
                     }
-                    for (const part of splitLong(businessContext, maxChars)) {
+                    for (const part of parts) {
                         chunks.push(buildBusinessContextChunk(table, part.text, part.suffix, schema.schemaId, table.sensitive || bizMentionsSensitive));
                     }
                 }
@@ -116,6 +162,9 @@ export function chunkSchema(sources, options = {}) {
         // Optional relationship chunks
         if (emitRelationships && table.relationships) {
             for (const rel of table.relationships) {
+                if (isUntrackedId(rel.from, schema) || isUntrackedId(rel.to, schema)) {
+                    continue;
+                }
                 const chunk = buildRelationshipChunk(table, rel, schema, schema.schemaId);
                 if (chunk.sensitive && !includeSensitive) {
                     stats.sensitiveExcluded++;
@@ -127,23 +176,44 @@ export function chunkSchema(sources, options = {}) {
             }
         }
     }
-    // Concept chunks. Concepts that link to sensitive columns include link
-    // metadata but their description is filtered if it names a sensitive col.
+    // Concept chunks. A concept that links to a sensitive column/table, or
+    // whose label/synonyms/description names a sensitive column (matched
+    // case-insensitively, like @askdb/enrich), is excluded by default.
     if (concepts?.frontmatter.concepts) {
-        const allSensitiveColumnNames = collectSensitiveColumnNames(schema);
         for (const concept of concepts.frontmatter.concepts) {
-            const conceptResult = buildConceptChunk(concept, schema.schemaId, schema, allSensitiveColumnNames, includeSensitive);
-            if (conceptResult.excluded) {
+            const conceptResult = buildConceptChunk(concept, schema.schemaId, schema, sensitiveMentions);
+            if (conceptResult.sensitive && !includeSensitive) {
                 stats.sensitiveExcluded++;
-                if (includeSensitive) {
-                    stats.sensitiveIncluded++;
-                    chunks.push(conceptResult.chunk);
-                }
+                continue;
             }
-            else {
-                if (conceptResult.sensitive)
-                    stats.sensitiveIncluded++;
-                chunks.push(conceptResult.chunk);
+            if (conceptResult.sensitive)
+                stats.sensitiveIncluded++;
+            chunks.push(conceptResult.chunk);
+        }
+    }
+    // Tenant policy body chunks — one per H2 section. The policy is
+    // schema-level prose, so like concepts it is checked against every
+    // sensitive column in the schema; a section naming one is excluded by
+    // default.
+    if (sources.tenantPolicy) {
+        for (const section of tenantPolicySections(sources.tenantPolicy)) {
+            const sensitive = sensitiveMentions.mentionsAny(section.body);
+            const parts = splitLong(section.body, maxChars);
+            if (sensitive && !includeSensitive) {
+                stats.sensitiveExcluded += parts.length;
+                continue;
+            }
+            if (sensitive)
+                stats.sensitiveIncluded += parts.length;
+            for (const part of parts) {
+                chunks.push({
+                    id: chunkId(schema.schemaId, `tenant-policy#${section.slug}${part.suffix}`),
+                    type: "tenant-policy",
+                    text: `${section.heading}\n${part.text}`,
+                    schemaId: schema.schemaId,
+                    refs: [],
+                    sensitive,
+                });
             }
         }
     }
@@ -158,21 +228,28 @@ export function chunkSchema(sources, options = {}) {
     return { chunks, stats };
 }
 // ---------- chunk builders ----------
-function buildTableChunk(table, schemaId, includeSensitive, schema) {
+function buildTableChunk(table, schemaId, includeSensitive, schema, filter) {
     const lines = [];
     const qualified = `${table.schema}.${table.name}`;
+    // Describable text that names a sensitive column is dropped by default and
+    // flags the chunk as sensitive when kept in opt-in mode.
+    const { allow } = filter;
     lines.push(`# ${qualified}`);
     if (!table.sensitive) {
-        if (table.description)
+        if (allow(table.description))
             lines.push(table.description);
-        if (table.aliases?.length)
-            lines.push(`Aliases: ${table.aliases.join(", ")}`);
-        if (table.primaryEntity)
+        const aliases = (table.aliases ?? []).filter((a) => allow(a));
+        if (aliases.length)
+            lines.push(`Aliases: ${aliases.join(", ")}`);
+        if (allow(table.primaryEntity))
             lines.push(`Primary entity: ${table.primaryEntity}`);
     }
     if (table.relationships?.length) {
         lines.push("Relationships:");
         for (const r of table.relationships) {
+            if (isUntrackedId(r.from, schema) || isUntrackedId(r.to, schema)) {
+                continue;
+            }
             if (!includeSensitive && (isSensitiveId(r.from, schema) || isSensitiveId(r.to, schema))) {
                 continue;
             }
@@ -190,31 +267,36 @@ function buildTableChunk(table, schemaId, includeSensitive, schema) {
             flags.push("PK");
         flags.push(c.nullable ? "NULL" : "NOT NULL");
         const tag = c.sensitive || table.sensitive ? " (sensitive)" : "";
-        const desc = !table.sensitive && !c.sensitive && c.description
+        const desc = !table.sensitive && !c.sensitive && allow(c.description)
             ? ` — ${c.description}`
             : "";
         lines.push(`- ${c.name} ${c.type} (${flags.join(" ")})${tag}${desc}`);
     }
     return {
-        id: `chunk:${table.id}`,
+        id: chunkId(schemaId, table.id),
         type: "table",
         text: lines.join("\n").trim(),
         schemaId,
         refs: [table.id, ...referencedColumns],
-        sensitive: table.sensitive,
+        sensitive: table.sensitive || filter.included,
     };
 }
-function buildColumnChunk(table, col, schemaId, columnNote, includeSensitive) {
-    const sensitive = col.sensitive || table.sensitive;
+/**
+ * @param includeDescribable emit description / aliases / enum values / note.
+ * @param describableSensitive the emitted describable layer references
+ *   sensitive content (opt-in mode only); sets `chunk.sensitive`.
+ */
+function buildColumnChunk(table, col, schemaId, columnNote, includeDescribable, describableSensitive) {
+    const columnSensitive = col.sensitive || table.sensitive;
     const flags = [];
     if (col.primaryKey)
         flags.push("PK");
     flags.push(col.nullable ? "NULL" : "NOT NULL");
     const lines = [];
     const qualified = `${table.schema}.${table.name}.${col.name}`;
-    lines.push(`Column: ${qualified} ${col.type} (${flags.join(" ")})${sensitive ? " (sensitive)" : ""}`);
+    lines.push(`Column: ${qualified} ${col.type} (${flags.join(" ")})${columnSensitive ? " (sensitive)" : ""}`);
     lines.push(`Id: ${col.id}`);
-    if (!sensitive || includeSensitive) {
+    if (includeDescribable) {
         if (col.description)
             lines.push(col.description);
         if (col.aliases?.length)
@@ -225,20 +307,30 @@ function buildColumnChunk(table, col, schemaId, columnNote, includeSensitive) {
             lines.push(`Note: ${columnNote}`);
     }
     return {
-        id: `chunk:${col.id}`,
+        id: chunkId(schemaId, col.id),
         type: "column",
         text: lines.join("\n").trim(),
         schemaId,
         refs: [table.id, col.id],
-        sensitive,
+        sensitive: columnSensitive || (includeDescribable && describableSensitive),
     };
 }
-function buildCqlChunk(table, body, suffix, schemaId, sensitive) {
-    const aliasNote = table.aliases?.length
-        ? ` (also: ${table.aliases.join(", ")})`
-        : "";
+function columnDescribableTexts(col, columnNote) {
+    const out = [];
+    if (col.description)
+        out.push(col.description);
+    if (col.aliases?.length)
+        out.push(...col.aliases);
+    if (col.enum?.length)
+        out.push(...col.enum.map(String));
+    if (columnNote)
+        out.push(columnNote);
+    return out;
+}
+function buildCqlChunk(table, aliases, body, suffix, schemaId, sensitive) {
+    const aliasNote = aliases.length ? ` (also: ${aliases.join(", ")})` : "";
     const text = `# ${table.schema}.${table.name}${aliasNote} — common query language\n${body.trim()}`;
-    const id = `chunk:${table.id}#cql${suffix}`;
+    const id = chunkId(schemaId, `${table.id}#cql${suffix}`);
     return {
         id,
         type: "cql",
@@ -248,11 +340,11 @@ function buildCqlChunk(table, body, suffix, schemaId, sensitive) {
         sensitive,
     };
 }
-function buildQuestionChunk(table, question, index, schemaId, sensitive) {
-    const entity = table.primaryEntity ? ` [${table.primaryEntity}]` : "";
+function buildQuestionChunk(table, primaryEntity, question, index, schemaId, sensitive) {
+    const entity = primaryEntity ? ` [${primaryEntity}]` : "";
     const text = `# ${table.schema}.${table.name}${entity} — example question\n${question.trim()}`;
     return {
-        id: `chunk:${table.id}#q:${index}`,
+        id: chunkId(schemaId, `${table.id}#q:${index}`),
         type: "question",
         text,
         schemaId,
@@ -263,7 +355,7 @@ function buildQuestionChunk(table, question, index, schemaId, sensitive) {
 function buildBusinessContextChunk(table, body, suffix, schemaId, sensitive) {
     const text = `# ${table.schema}.${table.name} — business context\n${body.trim()}`;
     return {
-        id: `chunk:${table.id}#biz${suffix}`,
+        id: chunkId(schemaId, `${table.id}#biz${suffix}`),
         type: "table",
         text,
         schemaId,
@@ -279,9 +371,13 @@ function buildRelationshipChunk(table, rel, schema, schemaId) {
     const fromName = fromTable ? `${fromTable.schema}.${fromTable.name}` : fromCol.tableId;
     const toName = toTable ? `${toTable.schema}.${toTable.name}` : toCol.tableId;
     const text = `Relationship: ${fromName}.${fromCol.column} references ${toName}.${toCol.column}`;
-    const sensitive = (fromTable?.sensitive ?? false) || (toTable?.sensitive ?? false);
+    // Sensitive when either side's table *or column* is sensitive.
+    const sensitive = (fromTable?.sensitive ?? false) ||
+        (toTable?.sensitive ?? false) ||
+        isSensitiveId(rel.from, schema) ||
+        isSensitiveId(rel.to, schema);
     return {
-        id: `chunk:${rel.from}->${rel.to}`,
+        id: chunkId(schemaId, `${rel.from}->${rel.to}`),
         type: "relationship",
         text,
         schemaId,
@@ -289,36 +385,81 @@ function buildRelationshipChunk(table, rel, schema, schemaId) {
         sensitive,
     };
 }
-function buildConceptChunk(concept, schemaId, schema, allSensitiveColumnNames, includeSensitive) {
-    const linkedSensitive = (concept.links ?? []).some((id) => isSensitiveId(id, schema));
-    const descriptionMentionsSensitive = concept.description
-        ? mentionsAnyName(concept.description, allSensitiveColumnNames)
-        : false;
+function buildConceptChunk(concept, schemaId, schema, sensitiveMentions) {
+    const links = (concept.links ?? []).filter((id) => !isUntrackedId(id, schema));
+    const linkedSensitive = links.some((id) => isSensitiveId(id, schema));
+    const textMentionsSensitive = [
+        concept.label,
+        ...(concept.synonyms ?? []),
+        concept.description ?? "",
+    ].some((text) => sensitiveMentions.mentionsAny(text));
+    const sensitive = linkedSensitive || textMentionsSensitive;
     const lines = [];
     lines.push(`# Concept: ${concept.label}`);
     lines.push(`Id: ${concept.id}`);
     if (concept.synonyms?.length)
         lines.push(`Synonyms: ${concept.synonyms.join(", ")}`);
-    if (concept.links?.length)
-        lines.push(`Links: ${concept.links.join(", ")}`);
-    const excludeForSensitive = (linkedSensitive || descriptionMentionsSensitive) && !includeSensitive;
-    if (concept.description && (!descriptionMentionsSensitive || includeSensitive)) {
+    if (links.length)
+        lines.push(`Links: ${links.join(", ")}`);
+    if (concept.description)
         lines.push(concept.description);
-    }
     return {
         chunk: {
-            id: `chunk:${concept.id}`,
+            id: chunkId(schemaId, concept.id),
             type: "concept",
             text: lines.join("\n").trim(),
             schemaId,
-            refs: [concept.id, ...(concept.links ?? [])],
-            sensitive: linkedSensitive || descriptionMentionsSensitive,
+            refs: [concept.id, ...links],
+            sensitive,
+        },
+        sensitive,
+    };
+}
+/** Tenant-policy sections to chunk: one per non-empty H2, else the whole body. */
+function tenantPolicySections(tenantPolicy) {
+    const out = [];
+    for (const [sectionName, sectionBody] of Object.entries(tenantPolicy.sections)) {
+        if (!sectionBody?.trim())
+            continue;
+        out.push({
+            slug: sectionName.toLowerCase().replace(/[^a-z0-9]+/g, "-"),
+            heading: `# Tenant policy — ${sectionName}`,
+            body: sectionBody.trim(),
+        });
+    }
+    if (out.length === 0 && tenantPolicy.body.trim()) {
+        out.push({ slug: "body", heading: "# Tenant policy", body: tenantPolicy.body.trim() });
+    }
+    return out;
+}
+/**
+ * Per-chunk gate for describable text (descriptions, aliases, primary
+ * entity) that the chunker drops piece by piece instead of excluding the
+ * whole chunk.
+ */
+function describableFilter(sensitiveMentions, includeSensitive) {
+    let dropped = false;
+    let included = false;
+    return {
+        allow(text) {
+            if (!text)
+                return false;
+            if (!sensitiveMentions.mentionsAny(text))
+                return true;
+            if (includeSensitive)
+                included = true;
+            else
+                dropped = true;
+            return includeSensitive;
+        },
+        get dropped() {
+            return dropped;
+        },
+        get included() {
+            return included;
         },
-        sensitive: linkedSensitive || descriptionMentionsSensitive,
-        excluded: excludeForSensitive,
     };
 }
-// ---------- helpers ----------
 function readColumnNote(md, columnName) {
     const body = md?.sections["Column notes"];
     if (!body)
@@ -351,20 +492,6 @@ function extractExampleQuestions(md) {
     }
     return out;
 }
-function mentionsAnyName(text, names) {
-    if (!text || names.length === 0)
-        return false;
-    for (const name of names) {
-        // Word-boundary match (also matches when wrapped in backticks).
-        const pattern = new RegExp(`(^|[^a-zA-Z0-9_])${escapeRegex(name)}([^a-zA-Z0-9_]|$)`);
-        if (pattern.test(text))
-            return true;
-    }
-    return false;
-}
-function escapeRegex(s) {
-    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
 /** Split a long body on paragraph boundaries; suffix is `""` for single-chunk, `#bc:N` (1-indexed) otherwise. */
 function splitLong(body, maxChars) {
     const trimmed = body.trim();
@@ -402,16 +529,6 @@ function parseColumnId(columnId) {
         column: columnId.slice(hashIdx + 1),
     };
 }
-function collectSensitiveColumnNames(schema) {
-    const names = new Set();
-    for (const t of schema.tables) {
-        for (const c of t.columns) {
-            if (c.sensitive || t.sensitive)
-                names.add(c.name);
-        }
-    }
-    return Array.from(names);
-}
 function isSensitiveId(id, schema) {
     for (const t of schema.tables) {
         if (t.id === id)
@@ -423,4 +540,9 @@ function isSensitiveId(id, schema) {
     }
     return false;
 }
+function isUntrackedId(id, schema) {
+    const tableId = id.includes("#") ? parseColumnId(id).tableId : id;
+    const table = schema.tables.find((t) => t.id === tableId);
+    return table?.tracked === false;
+}
 //# sourceMappingURL=chunker.js.map