@kaelio/ktx 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/assets/python/{kaelio_ktx-0.13.0-py3-none-any.whl → kaelio_ktx-0.14.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-program.js +1 -1
  5. package/dist/commands/ingest-commands.d.ts +9 -0
  6. package/dist/commands/ingest-commands.js +37 -1
  7. package/dist/commands/knowledge-commands.js +3 -0
  8. package/dist/commands/setup-commands.js +14 -1
  9. package/dist/connection-drivers.d.ts +2 -0
  10. package/dist/connection-drivers.js +7 -3
  11. package/dist/connection.d.ts +3 -0
  12. package/dist/connection.js +27 -0
  13. package/dist/connectors/bigquery/connector.d.ts +17 -6
  14. package/dist/connectors/bigquery/connector.js +152 -76
  15. package/dist/connectors/bigquery/dialect.d.ts +2 -2
  16. package/dist/connectors/clickhouse/connector.d.ts +1 -0
  17. package/dist/connectors/clickhouse/connector.js +45 -16
  18. package/dist/connectors/clickhouse/dialect.d.ts +2 -2
  19. package/dist/connectors/mongodb/connector.d.ts +69 -0
  20. package/dist/connectors/mongodb/connector.js +307 -0
  21. package/dist/connectors/mongodb/dialect.d.ts +17 -0
  22. package/dist/connectors/mongodb/dialect.js +50 -0
  23. package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
  24. package/dist/connectors/mongodb/live-database-introspection.js +24 -0
  25. package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
  26. package/dist/connectors/mongodb/schema-inference.js +110 -0
  27. package/dist/connectors/mysql/connector.d.ts +1 -0
  28. package/dist/connectors/mysql/connector.js +18 -2
  29. package/dist/connectors/mysql/dialect.d.ts +2 -2
  30. package/dist/connectors/postgres/connector.d.ts +1 -0
  31. package/dist/connectors/postgres/connector.js +20 -3
  32. package/dist/connectors/postgres/dialect.d.ts +2 -2
  33. package/dist/connectors/snowflake/connector.d.ts +1 -0
  34. package/dist/connectors/snowflake/connector.js +27 -3
  35. package/dist/connectors/snowflake/dialect.d.ts +2 -2
  36. package/dist/connectors/sqlite/connector.d.ts +11 -1
  37. package/dist/connectors/sqlite/connector.js +120 -17
  38. package/dist/connectors/sqlite/dialect.d.ts +2 -2
  39. package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
  40. package/dist/connectors/sqlite/read-query-child.js +23 -0
  41. package/dist/connectors/sqlserver/connector.d.ts +2 -0
  42. package/dist/connectors/sqlserver/connector.js +23 -5
  43. package/dist/connectors/sqlserver/dialect.d.ts +2 -2
  44. package/dist/context/cache/content-result-cache.d.ts +36 -0
  45. package/dist/context/cache/content-result-cache.js +14 -0
  46. package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
  47. package/dist/context/cache/sqlite-content-result-cache.js +223 -0
  48. package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
  49. package/dist/context/connections/bigquery-identifiers.js +7 -0
  50. package/dist/context/connections/configured-connections.d.ts +9 -0
  51. package/dist/context/connections/configured-connections.js +18 -0
  52. package/dist/context/connections/dialects.d.ts +30 -4
  53. package/dist/context/connections/dialects.js +38 -11
  54. package/dist/context/connections/drivers.js +21 -0
  55. package/dist/context/connections/gdrive-config.d.ts +19 -0
  56. package/dist/context/connections/gdrive-config.js +57 -0
  57. package/dist/context/connections/query-deadline.d.ts +31 -0
  58. package/dist/context/connections/query-deadline.js +37 -0
  59. package/dist/context/connections/read-only-sql.d.ts +5 -0
  60. package/dist/context/connections/read-only-sql.js +139 -1
  61. package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
  62. package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
  63. package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
  64. package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
  65. package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
  66. package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
  67. package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
  68. package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
  69. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
  70. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
  71. package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
  72. package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
  73. package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
  74. package/dist/context/ingest/adapters/gdrive/types.js +36 -0
  75. package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
  76. package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
  77. package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
  78. package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
  79. package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
  80. package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
  81. package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
  82. package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
  83. package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
  84. package/dist/context/ingest/adapters/live-database/stage.js +5 -5
  85. package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
  86. package/dist/context/ingest/adapters/metabase/types.js +1 -1
  87. package/dist/context/ingest/artifact-gates.d.ts +32 -1
  88. package/dist/context/ingest/artifact-gates.js +85 -18
  89. package/dist/context/ingest/final-gate-prune.d.ts +35 -0
  90. package/dist/context/ingest/final-gate-prune.js +229 -0
  91. package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
  92. package/dist/context/ingest/ingest-bundle.runner.js +459 -240
  93. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
  94. package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
  95. package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
  96. package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
  97. package/dist/context/ingest/local-adapters.js +6 -0
  98. package/dist/context/ingest/local-bundle-runtime.js +7 -2
  99. package/dist/context/ingest/local-ingest.js +1 -1
  100. package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
  101. package/dist/context/ingest/local-stage-ingest.js +3 -1
  102. package/dist/context/ingest/ports.d.ts +3 -0
  103. package/dist/context/ingest/report-snapshot.js +13 -3
  104. package/dist/context/ingest/reports.d.ts +3 -3
  105. package/dist/context/ingest/reports.js +3 -1
  106. package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
  107. package/dist/context/ingest/stages/build-wu-context.js +2 -1
  108. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
  109. package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
  110. package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
  111. package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
  112. package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
  113. package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
  114. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
  115. package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
  116. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
  117. package/dist/context/ingest/types.d.ts +3 -0
  118. package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
  119. package/dist/context/ingest/wiki-body-refs.js +37 -8
  120. package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
  121. package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
  122. package/dist/context/ingest/work-unit-cache.d.ts +67 -0
  123. package/dist/context/ingest/work-unit-cache.js +230 -0
  124. package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
  125. package/dist/context/llm/ai-sdk-runtime.js +5 -0
  126. package/dist/context/llm/claude-code-runtime.d.ts +4 -1
  127. package/dist/context/llm/claude-code-runtime.js +38 -8
  128. package/dist/context/llm/codex-runtime.d.ts +4 -1
  129. package/dist/context/llm/codex-runtime.js +48 -35
  130. package/dist/context/llm/runtime-port.d.ts +26 -0
  131. package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
  132. package/dist/context/llm/subprocess-generate-object-child.js +34 -0
  133. package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
  134. package/dist/context/llm/subprocess-generate-object.js +122 -0
  135. package/dist/context/mcp/context-tools.d.ts +2 -0
  136. package/dist/context/mcp/context-tools.js +111 -17
  137. package/dist/context/mcp/local-project-ports.d.ts +7 -0
  138. package/dist/context/mcp/local-project-ports.js +29 -0
  139. package/dist/context/mcp/logger.d.ts +24 -0
  140. package/dist/context/mcp/logger.js +49 -0
  141. package/dist/context/mcp/server.js +2 -0
  142. package/dist/context/mcp/types.d.ts +17 -0
  143. package/dist/context/memory/memory-agent.service.js +1 -1
  144. package/dist/context/project/config.d.ts +42 -0
  145. package/dist/context/project/config.js +5 -0
  146. package/dist/context/project/driver-schemas.d.ts +20 -0
  147. package/dist/context/project/driver-schemas.js +50 -1
  148. package/dist/context/project/project.js +1 -1
  149. package/dist/context/project/setup-config.js +1 -0
  150. package/dist/context/scan/description-generation.d.ts +4 -0
  151. package/dist/context/scan/description-generation.js +100 -13
  152. package/dist/context/scan/enabled-tables.d.ts +5 -0
  153. package/dist/context/scan/enabled-tables.js +13 -1
  154. package/dist/context/scan/enrichment-state.d.ts +50 -7
  155. package/dist/context/scan/enrichment-state.js +30 -13
  156. package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
  157. package/dist/context/scan/local-enrichment-artifacts.js +155 -32
  158. package/dist/context/scan/local-enrichment.d.ts +39 -4
  159. package/dist/context/scan/local-enrichment.js +345 -92
  160. package/dist/context/scan/local-scan.d.ts +4 -1
  161. package/dist/context/scan/local-scan.js +54 -13
  162. package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
  163. package/dist/context/scan/local-structural-artifacts.js +5 -0
  164. package/dist/context/scan/object-introspection.d.ts +21 -0
  165. package/dist/context/scan/object-introspection.js +35 -0
  166. package/dist/context/scan/relationship-benchmarks.js +5 -3
  167. package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
  168. package/dist/context/scan/relationship-composite-candidates.js +13 -1
  169. package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
  170. package/dist/context/scan/relationship-detection-budget.js +53 -0
  171. package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
  172. package/dist/context/scan/relationship-diagnostics.js +2 -0
  173. package/dist/context/scan/relationship-discovery.d.ts +9 -3
  174. package/dist/context/scan/relationship-discovery.js +27 -4
  175. package/dist/context/scan/relationship-llm-proposal.js +36 -27
  176. package/dist/context/scan/relationship-profiling.d.ts +7 -3
  177. package/dist/context/scan/relationship-profiling.js +53 -41
  178. package/dist/context/scan/relationship-validation.d.ts +6 -4
  179. package/dist/context/scan/relationship-validation.js +46 -32
  180. package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
  181. package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
  182. package/dist/context/scan/types.d.ts +5 -3
  183. package/dist/context/scan/types.js +2 -0
  184. package/dist/context/sl/local-query.js +5 -0
  185. package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
  186. package/dist/context/sl/semantic-layer.service.js +1 -1
  187. package/dist/context/sl/source-files.d.ts +5 -0
  188. package/dist/context/sl/source-files.js +17 -11
  189. package/dist/context/sl/tools/connection-id-schema.js +1 -1
  190. package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
  191. package/dist/context/sql-analysis/dialect-notes.js +40 -0
  192. package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
  193. package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
  194. package/dist/context/sql-analysis/dialects/mysql.md +9 -0
  195. package/dist/context/sql-analysis/dialects/postgres.md +10 -0
  196. package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
  197. package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
  198. package/dist/context/sql-analysis/dialects/tsql.md +10 -0
  199. package/dist/context/wiki/keys.js +1 -1
  200. package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
  201. package/dist/context/wiki/knowledge-wiki.service.js +2 -1
  202. package/dist/context/wiki/local-knowledge.d.ts +13 -0
  203. package/dist/context/wiki/local-knowledge.js +50 -6
  204. package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
  205. package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
  206. package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
  207. package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
  208. package/dist/context/wiki/types.d.ts +6 -0
  209. package/dist/context-build-view.d.ts +3 -0
  210. package/dist/context-build-view.js +1 -0
  211. package/dist/knowledge.d.ts +2 -0
  212. package/dist/knowledge.js +12 -1
  213. package/dist/local-adapters.js +11 -0
  214. package/dist/mcp-http-server.js +15 -1
  215. package/dist/mcp-server-factory.d.ts +2 -0
  216. package/dist/mcp-server-factory.js +15 -1
  217. package/dist/mcp-stdio-server.js +10 -2
  218. package/dist/notion-page-picker.js +1 -1
  219. package/dist/prompts/memory_agent_external_ingest.md +2 -0
  220. package/dist/public-ingest.d.ts +3 -1
  221. package/dist/public-ingest.js +3 -0
  222. package/dist/scan.d.ts +3 -1
  223. package/dist/scan.js +7 -0
  224. package/dist/setup-databases.d.ts +1 -1
  225. package/dist/setup-databases.js +43 -1
  226. package/dist/setup-sources.d.ts +5 -1
  227. package/dist/setup-sources.js +124 -48
  228. package/dist/setup.d.ts +3 -0
  229. package/dist/setup.js +6 -1
  230. package/dist/skills/analytics/SKILL.md +200 -2
  231. package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
  232. package/dist/skills/wiki_capture/SKILL.md +24 -0
  233. package/dist/sql.js +4 -0
  234. package/dist/status-project.d.ts +4 -0
  235. package/dist/status-project.js +54 -2
  236. package/dist/telemetry/events.d.ts +2 -2
  237. package/dist/text-ingest.d.ts +4 -0
  238. package/dist/text-ingest.js +58 -29
  239. package/dist/verbatim-ingest.d.ts +46 -0
  240. package/dist/verbatim-ingest.js +193 -0
  241. package/package.json +5 -1
  242. package/dist/context/ingest/final-gate-repair.d.ts +0 -28
  243. package/dist/context/ingest/final-gate-repair.js +0 -91
@@ -0,0 +1,97 @@
1
+ ---
2
+ name: gdrive_synthesize
3
+ description: Synthesize durable KTX wiki pages from staged Google Drive document pulls. Load when a WorkUnit contains Google Doc raw files from `docs/**`.
4
+ callers: [memory_agent]
5
+ ---
6
+
7
+ # Google Drive Doc Synthesis
8
+
9
+ Use this skill when a WorkUnit contains staged Google Drive content from `docs/**`.
10
+
11
+ ## Role
12
+
13
+ Each WorkUnit is one Google Doc plus its metadata. Read the assigned raw files, then write a small set of durable wiki entries that capture reusable organizational knowledge. Write final memory directly; do not write candidates.
14
+
15
+ ## Required Workflow
16
+
17
+ 1. Read the WorkUnit notes and `rawFiles` list. Document content lives in `page.md`; `metadata.json` holds title, path, url, modified time, and Drive folder context.
18
+ 2. For each assigned doc, call `read_raw_file`, or `read_raw_span` for oversized docs when the notes specify a span.
19
+ 3. Search `wiki_search` for existing pages that overlap the WorkUnit topics. Prefer updating an existing page over creating a duplicate.
20
+ 4. Use `context_evidence_search`, `context_evidence_read`, and `context_evidence_neighbors` when indexed document chunks would help reconcile related facts. Pass `chunkId` and `documentId` values verbatim as returned by the evidence tools.
21
+ 5. Write durable business knowledge with `wiki_write`. Aim for a small number of high-quality pages per doc. Include `rawPaths` with the exact Google Drive raw files that support each page.
22
+ 6. If a doc references warehouse, dbt, Looker, Metabase, or MetricFlow objects, you may verify them with `discover_data`, `entity_details`, `sql_execution`, `sl_discover`, or `sl_read_source`, but Google Drive docs are knowledge-only in v1. Do not create semantic-layer sources under the `gdrive` connection.
23
+ 7. For every deleted raw path in the Eviction Set, call `eviction_list`, decide retention, then `emit_eviction_decision`. Do this even when no wiki write is needed.
24
+
25
+ ## What To Capture
26
+
27
+ Capture durable, reusable company knowledge:
28
+
29
+ - policies, workflows, process rules, ownership conventions, and operating procedures
30
+ - product definitions, business terminology, and organizational guidance
31
+ - source-of-truth statements, caveats, conflict notes, and supersession guidance
32
+ - cross-system aliases that connect doc terminology to warehouse, dbt, Looker, Metabase, or MetricFlow names
33
+
34
+ Skip noisy or transient content:
35
+
36
+ - brainstorming notes with no durable rule
37
+ - task lists, meeting scheduling details, and time-bounded status updates
38
+ - duplicate docs with no new fact
39
+ - shallow summaries that add no reusable policy or definition
40
+
41
+ ## Quality
42
+
43
+ Prefer fewer, stronger entries. Every wiki entry must cite at least one Google Doc using its title or path and last modified date when available. When evidence conflicts, write a conflict note inside the wiki page rather than choosing silently.
44
+
45
+ If one doc covers several related ideas, synthesize the shared durable rules instead of writing one thin page per paragraph. For oversized spans, read only the assigned span unless the WorkUnit explicitly asks for neighboring context.
46
+
47
+ Search existing wiki pages for the same `tables:` or `sl_refs:` frontmatter and for source-of-truth aliases before creating a new page. If an existing page already documents the same warehouse object or business concept, update it instead of creating a differently named duplicate.
48
+
49
+ ## Citation Style
50
+
51
+ ```md
52
+ ## Agentic Harness
53
+ - The harness provides the operational framework that turns an agent prototype into a production system.
54
+ - Source: Google Doc - Herness, last modified 2026-05-24.
55
+ - Conflict note: An older internal note uses a narrower definition focused only on tool wiring; treat the current Google Doc as the durable operating definition unless replaced explicitly.
56
+ ```
57
+
58
+ ## Semantic-Layer Rules
59
+
60
+ - Google Drive docs are knowledge-only in v1; keep durable output in wiki pages.
61
+ - Do not create semantic-layer sources under the `gdrive` connection.
62
+ - If a doc references an existing warehouse or semantic-layer object and you can verify it, you may attach `sl_refs` in wiki output after confirmation.
63
+ - If a doc mentions a table or source that cannot be verified, keep the identifier in wiki text as unverified or use `emit_unmapped_fallback` only when the missing physical object itself is the important durable fact.
64
+
65
+ ## Identifier Verification Protocol
66
+
67
+ Before writing a wiki page on any topic:
68
+
69
+ 1. `discover_data({query: "<topic>"})` - see what wikis, SL sources, and raw
70
+ tables already exist. Prefer updating existing pages over creating new ones.
71
+
72
+ Before emitting any `schema.table` or `schema.table.column` into a wiki body,
73
+ `tables:` frontmatter, `sl_refs`, or `emit_unmapped_fallback`:
74
+
75
+ 2. `entity_details({connectionId, targets: [{display: "<identifier>"}]})` -
76
+ confirm the identifier resolves; inspect native types, FK/PK, and
77
+ sampleValues.
78
+ 3. For literal values from the doc, such as status codes or plan tiers,
79
+ check whether they appear in `entity_details` sampleValues for the relevant
80
+ column. If sampleValues is short or the sample may have missed real values,
81
+ run a `sql_execution` probe with the same warehouse connection id:
82
+ `sql_execution({connectionId, sql: "SELECT DISTINCT <col> FROM <ref> LIMIT 50"})`.
83
+ 4. If the candidate identifier still does not resolve, do one of:
84
+ - Use `sql_execution({connectionId, sql: "SELECT 1 FROM <ref> LIMIT 0"})`.
85
+ If it errors, the identifier is fictional.
86
+ - Wrap the identifier in `[unverified - from <rawPath>]` in the wiki body,
87
+ citing the exact raw path that mentioned it.
88
+ - When recording `emit_unmapped_fallback` with `no_physical_table`, include
89
+ the failing probe error in `clarification`.
90
+ 5. Never copy `<schema>.<table>` placeholder strings from these instructions
91
+ into output.
92
+
93
+ ## Tools
94
+
95
+ Allowed: `read_raw_file`, `read_raw_span`, `wiki_search`, `wiki_read`, `wiki_write`, `discover_data`, `entity_details`, `sql_execution`, `sl_discover`, `sl_read_source`, `context_evidence_search`, `context_evidence_read`, `context_evidence_neighbors`, `emit_unmapped_fallback`, `eviction_list`, `emit_eviction_decision`.
96
+
97
+ Not allowed: `context_candidate_write`, `context_candidate_mark`, `sl_write_source`, `sl_edit_source`, `sl_validate`.
@@ -112,6 +112,30 @@ All three fields use REPLACE semantics on update:
112
112
  - Pass `[]` → field is cleared.
113
113
  - Pass `[values]` → replaces existing with exactly those values (no merging).
114
114
 
115
+ ## Connection scoping
116
+
117
+ A project may have several databases whose schemas reuse the same concept names
118
+ (two warehouses each with `orders`, `customers`, …). The `connections`
119
+ frontmatter field keeps database-specific pages from polluting searches about
120
+ other databases.
121
+
122
+ - The `wiki_write` tool accepts a `connections` field (list of connection ids,
123
+ same REPLACE semantics as `tags`). Absent or empty ⇒ the page is **unscoped**
124
+ and applies to every connection.
125
+ - When this ingest/turn is scoped to a connection (its id appears in the prompt
126
+ context — e.g. `connectionId: warehouse` in the SL Sources header or the
127
+ `<context>` block), set `connections: [<that id>]` on pages whose content is
128
+ **specific to that database** ("in this warehouse `user_id` is the device id,
129
+ not the account id"). Pair this with a connection-distinctive key so two
130
+ databases' same-concept pages can coexist: `orders_sales_db`, not `orders`.
131
+ - Leave `connections` empty for clearly **org-wide** knowledge ("fiscal year
132
+ starts in February") so it stays visible everywhere. Do not scope a page to a
133
+ connection just because the turn happened to be connection-scoped.
134
+ - Keys are still a flat, global namespace; `connections` does not namespace
135
+ them. A connection-scoped write whose key already belongs to a page scoped to
136
+ a *different* connection is rejected to prevent silently overwriting it — pick
137
+ a connection-distinctive key instead.
138
+
115
139
  ## Editing existing pages
116
140
 
117
141
  Two modes:
package/dist/sql.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import { FEDERATED_CONNECTION_ID } from './context/connections/federation.js';
2
2
  import { executeProjectReadOnlySql } from './context/connections/project-sql-executor.js';
3
+ import { assertSqlQueryableConnection } from './context/connections/dialects.js';
3
4
  import { resolveConfiguredConnection } from './context/connections/resolve-connection.js';
4
5
  import { loadKtxProject } from './context/project/project.js';
5
6
  import { sqlAnalysisDialectForDriver } from './context/sql-analysis/dialect.js';
@@ -90,6 +91,9 @@ export async function runKtxSql(args, io = process, deps = {}) {
90
91
  const connection = isFederated ? undefined : resolveConfiguredConnection(project.config, args.connectionId);
91
92
  driver = isFederated ? 'duckdb' : String(connection?.driver ?? 'unknown').toLowerCase();
92
93
  demoConnection = isFederated ? false : isDemoConnection(args.connectionId, connection);
94
+ if (!isFederated) {
95
+ assertSqlQueryableConnection(args.connectionId, connection?.driver);
96
+ }
93
97
  const createSqlAnalysis = deps.createSqlAnalysis ??
94
98
  (() => createManagedDaemonSqlAnalysisPort({
95
99
  cliVersion: args.cliVersion,
@@ -75,6 +75,10 @@ interface LocalStatsIngestPerConnection {
75
75
  connectionId: string;
76
76
  adapter: string;
77
77
  lastCompletedAt: string;
78
+ skippedObjects: Array<{
79
+ name: string;
80
+ reason: string;
81
+ }>;
78
82
  }
79
83
  interface LocalStatsSemanticLayerEntry {
80
84
  connectionId: string;
@@ -4,6 +4,7 @@ import { runClaudeCodeAuthProbe } from './context/llm/claude-code-runtime.js';
4
4
  import { CODEX_ISOLATION_WARNING, CODEX_ISOLATION_WARNING_FIX, } from './context/llm/codex-isolation.js';
5
5
  import { runCodexAuthProbe } from './context/llm/codex-runtime.js';
6
6
  import { ktxLocalStateDbPath } from './context/project/local-state-db.js';
7
+ import { listReferencedConnectionIds } from './context/wiki/local-knowledge.js';
7
8
  import { isQueryHistoryEnabled, queryHistoryDialectForConnection, } from './context/ingest/adapters/historic-sql/connection-dialect.js';
8
9
  import { historicSqlProbeCatalogName, runHistoricSqlReadinessProbe, } from './context/ingest/historic-sql-probes.js';
9
10
  import { formatClaudeCodePromptCachingFix, formatClaudeCodePromptCachingWarning, ignoredClaudeCodePromptCachingFields, } from './claude-code-prompt-caching.js';
@@ -387,6 +388,29 @@ function buildStorageStatus(config) {
387
388
  gitAuthor: config.storage.git.author,
388
389
  };
389
390
  }
391
+ /**
392
+ * Warn (never fail) when stored wiki pages reference connection ids that are no
393
+ * longer in `ktx.yaml`. Config and page content evolve independently, so a
394
+ * dangling reference is a soft condition — the pages still load, search, and
395
+ * read; it just signals a typo or a removed connection.
396
+ */
397
+ async function buildUnknownConnectionWarning(project) {
398
+ let referenced;
399
+ try {
400
+ referenced = await listReferencedConnectionIds(project);
401
+ }
402
+ catch {
403
+ return null;
404
+ }
405
+ const unknown = referenced.filter((id) => !Object.hasOwn(project.config.connections, id));
406
+ if (unknown.length === 0) {
407
+ return null;
408
+ }
409
+ return {
410
+ message: `Wiki pages reference connection id(s) not in ktx.yaml: ${unknown.join(', ')}. Those pages still load and search.`,
411
+ fix: 'Add the connection(s) via `ktx setup`, or update the pages’ `connections` frontmatter.',
412
+ };
413
+ }
390
414
  function buildWarnings(config, connections, llm, embeddings) {
391
415
  const warnings = [];
392
416
  for (const adapter of config.ingest.adapters) {
@@ -555,6 +579,21 @@ function tryQuery(run, fallback) {
555
579
  return fallback;
556
580
  }
557
581
  }
582
+ function skippedObjectsFromReportBody(bodyJson) {
583
+ try {
584
+ const body = JSON.parse(bodyJson);
585
+ const skipped = body.fetch?.skipped;
586
+ if (!Array.isArray(skipped))
587
+ return [];
588
+ return skipped.map((issue) => ({
589
+ name: typeof issue.entityId === 'string' && issue.entityId.length > 0 ? issue.entityId : 'object',
590
+ reason: typeof issue.message === 'string' ? issue.message : 'introspection failed',
591
+ }));
592
+ }
593
+ catch {
594
+ return [];
595
+ }
596
+ }
558
597
  /** @internal */
559
598
  export async function buildLocalStatsStatus(project) {
560
599
  const dbPath = ktxLocalStateDbPath(project);
@@ -581,8 +620,10 @@ export async function buildLocalStatsStatus(project) {
581
620
  const totalCompletedRuns = tryQuery(() => db
582
621
  .prepare(`SELECT COUNT(*) AS n FROM local_ingest_reports WHERE status = 'done'`)
583
622
  .get()?.n ?? 0, 0);
584
- const ingestRows = tryQuery(() => db
585
- .prepare(`SELECT connection_id, adapter, MAX(completed_at) AS last_completed_at
623
+ const ingestRows = tryQuery(() =>
624
+ // SQLite returns body_json from the MAX(completed_at) row for each group.
625
+ db
626
+ .prepare(`SELECT connection_id, adapter, MAX(completed_at) AS last_completed_at, body_json
586
627
  FROM local_ingest_reports
587
628
  WHERE status = 'done'
588
629
  GROUP BY connection_id, adapter`)
@@ -595,6 +636,7 @@ export async function buildLocalStatsStatus(project) {
595
636
  connectionId: row.connection_id,
596
637
  adapter: row.adapter,
597
638
  lastCompletedAt: row.last_completed_at,
639
+ skippedObjects: skippedObjectsFromReportBody(row.body_json),
598
640
  });
599
641
  }
600
642
  }
@@ -683,6 +725,10 @@ export async function buildProjectStatus(project, options = {}) {
683
725
  const queryHistory = await buildQueryHistoryStatus(project, options);
684
726
  const pipeline = buildPipelineStatus(config);
685
727
  const warnings = buildWarnings(config, connections, llm, embeddings);
728
+ const unknownConnectionWarning = await buildUnknownConnectionWarning(project);
729
+ if (unknownConnectionWarning) {
730
+ warnings.push(unknownConnectionWarning);
731
+ }
686
732
  const localStats = await buildLocalStatsStatus(project);
687
733
  const { verdict, reason, nextActions } = buildVerdict(llm, embeddings, connections, queryHistory, warnings);
688
734
  return {
@@ -796,6 +842,12 @@ function renderLocalStats(lines, stats, dim, bold) {
796
842
  const adapterWidth = Math.max(...ingest.perConnection.map((entry) => entry.adapter.length));
797
843
  for (const entry of ingest.perConnection) {
798
844
  lines.push(` ${entry.connectionId.padEnd(nameWidth)} ${dim(entry.adapter.padEnd(adapterWidth))} ${dim(`last ${formatRelativeFromNow(entry.lastCompletedAt)}`)}`);
845
+ if (entry.skippedObjects.length > 0) {
846
+ const first = entry.skippedObjects[0];
847
+ const extra = entry.skippedObjects.length - 1;
848
+ const detail = `${first.name}: ${first.reason}${extra > 0 ? ` (+${extra} more)` : ''}`;
849
+ lines.push(` ${' '.repeat(nameWidth)} ${dim(`${entry.skippedObjects.length} object${entry.skippedObjects.length === 1 ? '' : 's'} skipped — ${detail}`)}`);
850
+ }
799
851
  }
800
852
  }
801
853
  if (stats.wikiPages.length === 0) {
@@ -62,10 +62,10 @@ export declare const telemetryEventSchemas: {
62
62
  isCi: z.ZodBoolean;
63
63
  step: z.ZodEnum<{
64
64
  embeddings: "embeddings";
65
+ databases: "databases";
65
66
  project: "project";
66
67
  models: "models";
67
68
  sources: "sources";
68
- databases: "databases";
69
69
  context: "context";
70
70
  runtime: "runtime";
71
71
  secrets: "secrets";
@@ -254,9 +254,9 @@ export declare const telemetryEventSchemas: {
254
254
  queryVerb: z.ZodEnum<{
255
255
  show: "show";
256
256
  with: "with";
257
+ explain: "explain";
257
258
  select: "select";
258
259
  other: "other";
259
- explain: "explain";
260
260
  }>;
261
261
  referencedTableCount: z.ZodNumber;
262
262
  durationMs: z.ZodNumber;
@@ -2,6 +2,7 @@ import type { MemoryAgentInput } from './context/memory/types.js';
2
2
  import type { MemoryIngestStatus } from './context/memory/memory-runs.js';
3
3
  import { type KtxLocalProject } from './context/project/project.js';
4
4
  import type { KtxCliIo } from './cli-runtime.js';
5
+ import { type VerbatimIngestorPort } from './verbatim-ingest.js';
5
6
  export interface KtxTextIngestArgs {
6
7
  projectDir: string;
7
8
  texts: string[];
@@ -10,6 +11,8 @@ export interface KtxTextIngestArgs {
10
11
  userId: string;
11
12
  json: boolean;
12
13
  failFast: boolean;
14
+ /** Code-driven verbatim ingest: store the document body unchanged, LLM derives metadata only. */
15
+ verbatim?: boolean;
13
16
  }
14
17
  /** @internal */
15
18
  export interface TextMemoryIngestPort {
@@ -24,6 +27,7 @@ export interface KtxTextIngestDeps {
24
27
  projectDir: string;
25
28
  }) => Promise<KtxLocalProject>;
26
29
  createMemoryIngest?: (project: KtxLocalProject) => TextMemoryIngestPort;
30
+ createVerbatimIngestor?: (project: KtxLocalProject) => VerbatimIngestorPort;
27
31
  readFile?: (path: string) => Promise<string>;
28
32
  readStdin?: () => Promise<string>;
29
33
  now?: () => number;
@@ -4,11 +4,15 @@ import { createLocalProjectMemoryIngest } from './context/memory/local-memory.js
4
4
  import { loadKtxProject } from './context/project/project.js';
5
5
  import { createRepainter, initViewState, renderContextBuildView } from './context-build-view.js';
6
6
  import { formatDuration } from './demo-metrics.js';
7
+ import { createLocalProjectVerbatimIngestor, } from './verbatim-ingest.js';
7
8
  const INLINE_TEXT_LABEL_MAX_LENGTH = 50;
8
9
  const ANSI_ESCAPE_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g;
9
10
  function defaultCreateMemoryIngest(project) {
10
11
  return createLocalProjectMemoryIngest(project);
11
12
  }
13
+ function defaultCreateVerbatimIngestor(project) {
14
+ return createLocalProjectVerbatimIngestor(project);
15
+ }
12
16
  async function defaultReadStdin() {
13
17
  const chunks = [];
14
18
  process.stdin.setEncoding('utf-8');
@@ -72,17 +76,17 @@ async function loadItems(args, deps) {
72
76
  args.texts.forEach((content, index) => {
73
77
  const label = textLabel(content, index, usedTextLabels);
74
78
  usedTextLabels.add(label);
75
- items.push({ label, content });
79
+ items.push({ label, content, origin: { kind: 'text' } });
76
80
  });
77
81
  const readFile = deps.readFile ?? defaultReadFile;
78
82
  const readStdin = deps.readStdin ?? defaultReadStdin;
79
83
  for (const file of args.files) {
80
84
  if (file === '-') {
81
- items.push({ label: stdinLabel(items), content: await readStdin() });
85
+ items.push({ label: stdinLabel(items), content: await readStdin(), origin: { kind: 'stdin' } });
82
86
  }
83
87
  else {
84
88
  const path = resolve(file);
85
- items.push({ label: basename(path), content: await readFile(path) });
89
+ items.push({ label: basename(path), content: await readFile(path), origin: { kind: 'file', path } });
86
90
  }
87
91
  }
88
92
  return items;
@@ -112,13 +116,13 @@ function makeTarget(label) {
112
116
  function allTargets(state) {
113
117
  return [...state.primarySources, ...state.contextSources];
114
118
  }
115
- function renderTextIngestView(state, styled) {
119
+ function renderTextIngestView(state, styled, verbatim) {
116
120
  return renderContextBuildView(state, {
117
121
  styled,
118
- title: 'Ingesting text memory',
119
- contextGroupLabel: 'Texts',
120
- sourceIngestRunningText: 'capturing...',
121
- completedItemName: { singular: 'text', plural: 'texts' },
122
+ title: verbatim ? 'Writing verbatim pages' : 'Ingesting text memory',
123
+ contextGroupLabel: verbatim ? 'Documents' : 'Texts',
124
+ sourceIngestRunningText: verbatim ? 'writing...' : 'capturing...',
125
+ completedItemName: verbatim ? { singular: 'page', plural: 'pages' } : { singular: 'text', plural: 'texts' },
122
126
  });
123
127
  }
124
128
  function summarizeCaptured(captured) {
@@ -173,7 +177,9 @@ export async function runKtxTextIngest(args, io, deps = {}) {
173
177
  return 1;
174
178
  }
175
179
  const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir });
176
- const memoryIngest = (deps.createMemoryIngest ?? defaultCreateMemoryIngest)(project);
180
+ const isVerbatim = args.verbatim === true;
181
+ const verbatimIngestor = isVerbatim ? (deps.createVerbatimIngestor ?? defaultCreateVerbatimIngestor)(project) : null;
182
+ const memoryIngest = isVerbatim ? null : (deps.createMemoryIngest ?? defaultCreateMemoryIngest)(project);
177
183
  const now = deps.now ?? (() => Date.now());
178
184
  const batchId = now();
179
185
  const state = initViewState(items.map((item) => makeTarget(item.label)));
@@ -182,7 +188,7 @@ export async function runKtxTextIngest(args, io, deps = {}) {
182
188
  const repainter = isTTY ? createRepainter(io) : null;
183
189
  const results = [];
184
190
  state.startedAt = now();
185
- const paint = () => repainter?.paint(renderTextIngestView(state, true));
191
+ const paint = () => repainter?.paint(renderTextIngestView(state, true, isVerbatim));
186
192
  paint();
187
193
  let spinnerInterval = null;
188
194
  if (repainter) {
@@ -204,28 +210,50 @@ export async function runKtxTextIngest(args, io, deps = {}) {
204
210
  const target = targets[index];
205
211
  target.status = 'running';
206
212
  target.startedAt = now();
207
- target.detailLine = 'capturing...';
213
+ target.detailLine = isVerbatim ? 'writing...' : 'capturing...';
208
214
  target.progressUpdatedAtMs = target.startedAt;
209
215
  paint();
210
216
  let runId = null;
211
217
  let result;
212
218
  try {
213
- const ingestInput = {
214
- userId: args.userId,
215
- chatId: `cli-text-ingest-${batchId}-${index + 1}`,
216
- userMessage: `Ingest external text artifact ${artifactReference(item.label)} into ktx memory.`,
217
- assistantMessage: item.content.trim(),
218
- ...(args.connectionId ? { connectionId: args.connectionId } : {}),
219
- sourceType: 'external_ingest',
220
- };
221
- const ingest = await memoryIngest.ingest(ingestInput);
222
- runId = ingest.runId;
223
- await memoryIngest.waitForRun(runId);
224
- const status = await memoryIngest.status(runId);
225
- if (!status) {
226
- throw new Error(`Memory ingest run "${runId}" was not found.`);
219
+ if (verbatimIngestor) {
220
+ const verbatimItem = {
221
+ origin: item.origin,
222
+ content: item.content,
223
+ ...(args.connectionId ? { connectionId: args.connectionId } : {}),
224
+ };
225
+ const outcome = await verbatimIngestor.ingest(verbatimItem);
226
+ result = {
227
+ label: item.label,
228
+ runId: null,
229
+ status: 'done',
230
+ captured: { wiki: [outcome.pageKey], sl: [], xrefs: [] },
231
+ commitHash: outcome.commitHash,
232
+ error: null,
233
+ };
234
+ }
235
+ else {
236
+ // memoryIngest is set whenever verbatim is off — they are mutually exclusive.
237
+ if (!memoryIngest) {
238
+ throw new Error('Memory ingest was not initialized.');
239
+ }
240
+ const ingestInput = {
241
+ userId: args.userId,
242
+ chatId: `cli-text-ingest-${batchId}-${index + 1}`,
243
+ userMessage: `Ingest external text artifact ${artifactReference(item.label)} into ktx memory.`,
244
+ assistantMessage: item.content.trim(),
245
+ ...(args.connectionId ? { connectionId: args.connectionId } : {}),
246
+ sourceType: 'external_ingest',
247
+ };
248
+ const ingest = await memoryIngest.ingest(ingestInput);
249
+ runId = ingest.runId;
250
+ await memoryIngest.waitForRun(runId);
251
+ const status = await memoryIngest.status(runId);
252
+ if (!status) {
253
+ throw new Error(`Memory ingest run "${runId}" was not found.`);
254
+ }
255
+ result = resultFromStatus(item.label, status);
227
256
  }
228
- result = resultFromStatus(item.label, status);
229
257
  }
230
258
  catch (error) {
231
259
  result = errorResult(item.label, runId, error);
@@ -254,17 +282,18 @@ export async function runKtxTextIngest(args, io, deps = {}) {
254
282
  writeJsonResult(args, results, io);
255
283
  }
256
284
  else if (repainter) {
257
- repainter.paint(renderTextIngestView(state, true));
285
+ repainter.paint(renderTextIngestView(state, true, isVerbatim));
258
286
  writePlainFailures(results, io);
259
287
  }
260
288
  else {
261
- io.stdout.write(renderTextIngestView(state, false));
289
+ io.stdout.write(renderTextIngestView(state, false, isVerbatim));
262
290
  writePlainFailures(results, io);
263
291
  }
264
292
  if (!args.json && results.length > 0) {
265
293
  const duration = state.totalElapsedMs > 0 ? ` in ${formatDuration(state.totalElapsedMs)}` : '';
266
294
  const outcome = results.some((result) => result.status === 'error') ? 'finished with failures' : 'finished';
267
- io.stdout.write(`Text memory ingest ${outcome}${duration}.\n`);
295
+ const label = isVerbatim ? 'Verbatim ingest' : 'Text memory ingest';
296
+ io.stdout.write(`${label} ${outcome}${duration}.\n`);
268
297
  }
269
298
  return results.some((result) => result.status === 'error') ? 1 : 0;
270
299
  }
@@ -0,0 +1,46 @@
1
+ import type { KtxLlmRuntimePort } from './context/llm/runtime-port.js';
2
+ import type { KtxLocalProject } from './context/project/project.js';
3
+ import type { WikiFrontmatter } from './context/wiki/types.js';
4
+ import type { KtxEmbeddingProvider } from './llm/types.js';
5
+ export interface VerbatimIngestOrigin {
6
+ kind: 'file' | 'text' | 'stdin';
7
+ /** Present only for `kind: 'file'`; the resolved path the key basename is derived from. */
8
+ path?: string;
9
+ }
10
+ /** @internal */
11
+ export declare function splitInputDocument(raw: string): {
12
+ frontmatter: Record<string, unknown>;
13
+ body: string;
14
+ };
15
+ /** @internal */
16
+ export declare function deriveVerbatimPageKey(origin: VerbatimIngestOrigin, body: string): string;
17
+ /** @internal */
18
+ export declare function deriveDegradedSummary(body: string): string;
19
+ /** @internal */
20
+ export declare function buildVerbatimFrontmatter(input: {
21
+ inputFrontmatter: Record<string, unknown>;
22
+ summary: string;
23
+ tags: string[];
24
+ slRefs: string[];
25
+ connectionId?: string;
26
+ }): WikiFrontmatter & Record<string, unknown>;
27
+ export interface VerbatimIngestItem {
28
+ origin: VerbatimIngestOrigin;
29
+ content: string;
30
+ connectionId?: string;
31
+ }
32
+ export interface VerbatimIngestResult {
33
+ pageKey: string;
34
+ outcome: 'written' | 'unchanged';
35
+ connections: string[];
36
+ commitHash: string | null;
37
+ }
38
+ export interface VerbatimIngestorPort {
39
+ ingest(item: VerbatimIngestItem): Promise<VerbatimIngestResult>;
40
+ }
41
+ export interface CreateLocalProjectVerbatimIngestorDeps {
42
+ /** `undefined` ⇒ resolve from project config; `null` ⇒ force degraded (offline) metadata. */
43
+ llmRuntime?: KtxLlmRuntimePort | null;
44
+ embeddingProvider?: KtxEmbeddingProvider | null;
45
+ }
46
+ export declare function createLocalProjectVerbatimIngestor(project: KtxLocalProject, deps?: CreateLocalProjectVerbatimIngestorDeps): VerbatimIngestorPort;