@kaelio/ktx 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/assets/python/{kaelio_ktx-0.13.0-py3-none-any.whl → kaelio_ktx-0.14.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-program.js +1 -1
  5. package/dist/commands/ingest-commands.d.ts +9 -0
  6. package/dist/commands/ingest-commands.js +37 -1
  7. package/dist/commands/knowledge-commands.js +3 -0
  8. package/dist/commands/setup-commands.js +14 -1
  9. package/dist/connection-drivers.d.ts +2 -0
  10. package/dist/connection-drivers.js +7 -3
  11. package/dist/connection.d.ts +3 -0
  12. package/dist/connection.js +27 -0
  13. package/dist/connectors/bigquery/connector.d.ts +17 -6
  14. package/dist/connectors/bigquery/connector.js +152 -76
  15. package/dist/connectors/bigquery/dialect.d.ts +2 -2
  16. package/dist/connectors/clickhouse/connector.d.ts +1 -0
  17. package/dist/connectors/clickhouse/connector.js +45 -16
  18. package/dist/connectors/clickhouse/dialect.d.ts +2 -2
  19. package/dist/connectors/mongodb/connector.d.ts +69 -0
  20. package/dist/connectors/mongodb/connector.js +307 -0
  21. package/dist/connectors/mongodb/dialect.d.ts +17 -0
  22. package/dist/connectors/mongodb/dialect.js +50 -0
  23. package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
  24. package/dist/connectors/mongodb/live-database-introspection.js +24 -0
  25. package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
  26. package/dist/connectors/mongodb/schema-inference.js +110 -0
  27. package/dist/connectors/mysql/connector.d.ts +1 -0
  28. package/dist/connectors/mysql/connector.js +18 -2
  29. package/dist/connectors/mysql/dialect.d.ts +2 -2
  30. package/dist/connectors/postgres/connector.d.ts +1 -0
  31. package/dist/connectors/postgres/connector.js +20 -3
  32. package/dist/connectors/postgres/dialect.d.ts +2 -2
  33. package/dist/connectors/snowflake/connector.d.ts +1 -0
  34. package/dist/connectors/snowflake/connector.js +27 -3
  35. package/dist/connectors/snowflake/dialect.d.ts +2 -2
  36. package/dist/connectors/sqlite/connector.d.ts +11 -1
  37. package/dist/connectors/sqlite/connector.js +120 -17
  38. package/dist/connectors/sqlite/dialect.d.ts +2 -2
  39. package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
  40. package/dist/connectors/sqlite/read-query-child.js +23 -0
  41. package/dist/connectors/sqlserver/connector.d.ts +2 -0
  42. package/dist/connectors/sqlserver/connector.js +23 -5
  43. package/dist/connectors/sqlserver/dialect.d.ts +2 -2
  44. package/dist/context/cache/content-result-cache.d.ts +36 -0
  45. package/dist/context/cache/content-result-cache.js +14 -0
  46. package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
  47. package/dist/context/cache/sqlite-content-result-cache.js +223 -0
  48. package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
  49. package/dist/context/connections/bigquery-identifiers.js +7 -0
  50. package/dist/context/connections/configured-connections.d.ts +9 -0
  51. package/dist/context/connections/configured-connections.js +18 -0
  52. package/dist/context/connections/dialects.d.ts +30 -4
  53. package/dist/context/connections/dialects.js +38 -11
  54. package/dist/context/connections/drivers.js +21 -0
  55. package/dist/context/connections/gdrive-config.d.ts +19 -0
  56. package/dist/context/connections/gdrive-config.js +57 -0
  57. package/dist/context/connections/query-deadline.d.ts +31 -0
  58. package/dist/context/connections/query-deadline.js +37 -0
  59. package/dist/context/connections/read-only-sql.d.ts +5 -0
  60. package/dist/context/connections/read-only-sql.js +139 -1
  61. package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
  62. package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
  63. package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
  64. package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
  65. package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
  66. package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
  67. package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
  68. package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
  69. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
  70. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
  71. package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
  72. package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
  73. package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
  74. package/dist/context/ingest/adapters/gdrive/types.js +36 -0
  75. package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
  76. package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
  77. package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
  78. package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
  79. package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
  80. package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
  81. package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
  82. package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
  83. package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
  84. package/dist/context/ingest/adapters/live-database/stage.js +5 -5
  85. package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
  86. package/dist/context/ingest/adapters/metabase/types.js +1 -1
  87. package/dist/context/ingest/artifact-gates.d.ts +32 -1
  88. package/dist/context/ingest/artifact-gates.js +85 -18
  89. package/dist/context/ingest/final-gate-prune.d.ts +35 -0
  90. package/dist/context/ingest/final-gate-prune.js +229 -0
  91. package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
  92. package/dist/context/ingest/ingest-bundle.runner.js +459 -240
  93. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
  94. package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
  95. package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
  96. package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
  97. package/dist/context/ingest/local-adapters.js +6 -0
  98. package/dist/context/ingest/local-bundle-runtime.js +7 -2
  99. package/dist/context/ingest/local-ingest.js +1 -1
  100. package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
  101. package/dist/context/ingest/local-stage-ingest.js +3 -1
  102. package/dist/context/ingest/ports.d.ts +3 -0
  103. package/dist/context/ingest/report-snapshot.js +13 -3
  104. package/dist/context/ingest/reports.d.ts +3 -3
  105. package/dist/context/ingest/reports.js +3 -1
  106. package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
  107. package/dist/context/ingest/stages/build-wu-context.js +2 -1
  108. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
  109. package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
  110. package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
  111. package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
  112. package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
  113. package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
  114. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
  115. package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
  116. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
  117. package/dist/context/ingest/types.d.ts +3 -0
  118. package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
  119. package/dist/context/ingest/wiki-body-refs.js +37 -8
  120. package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
  121. package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
  122. package/dist/context/ingest/work-unit-cache.d.ts +67 -0
  123. package/dist/context/ingest/work-unit-cache.js +230 -0
  124. package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
  125. package/dist/context/llm/ai-sdk-runtime.js +5 -0
  126. package/dist/context/llm/claude-code-runtime.d.ts +4 -1
  127. package/dist/context/llm/claude-code-runtime.js +38 -8
  128. package/dist/context/llm/codex-runtime.d.ts +4 -1
  129. package/dist/context/llm/codex-runtime.js +48 -35
  130. package/dist/context/llm/runtime-port.d.ts +26 -0
  131. package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
  132. package/dist/context/llm/subprocess-generate-object-child.js +34 -0
  133. package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
  134. package/dist/context/llm/subprocess-generate-object.js +122 -0
  135. package/dist/context/mcp/context-tools.d.ts +2 -0
  136. package/dist/context/mcp/context-tools.js +111 -17
  137. package/dist/context/mcp/local-project-ports.d.ts +7 -0
  138. package/dist/context/mcp/local-project-ports.js +29 -0
  139. package/dist/context/mcp/logger.d.ts +24 -0
  140. package/dist/context/mcp/logger.js +49 -0
  141. package/dist/context/mcp/server.js +2 -0
  142. package/dist/context/mcp/types.d.ts +17 -0
  143. package/dist/context/memory/memory-agent.service.js +1 -1
  144. package/dist/context/project/config.d.ts +42 -0
  145. package/dist/context/project/config.js +5 -0
  146. package/dist/context/project/driver-schemas.d.ts +20 -0
  147. package/dist/context/project/driver-schemas.js +50 -1
  148. package/dist/context/project/project.js +1 -1
  149. package/dist/context/project/setup-config.js +1 -0
  150. package/dist/context/scan/description-generation.d.ts +4 -0
  151. package/dist/context/scan/description-generation.js +100 -13
  152. package/dist/context/scan/enabled-tables.d.ts +5 -0
  153. package/dist/context/scan/enabled-tables.js +13 -1
  154. package/dist/context/scan/enrichment-state.d.ts +50 -7
  155. package/dist/context/scan/enrichment-state.js +30 -13
  156. package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
  157. package/dist/context/scan/local-enrichment-artifacts.js +155 -32
  158. package/dist/context/scan/local-enrichment.d.ts +39 -4
  159. package/dist/context/scan/local-enrichment.js +345 -92
  160. package/dist/context/scan/local-scan.d.ts +4 -1
  161. package/dist/context/scan/local-scan.js +54 -13
  162. package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
  163. package/dist/context/scan/local-structural-artifacts.js +5 -0
  164. package/dist/context/scan/object-introspection.d.ts +21 -0
  165. package/dist/context/scan/object-introspection.js +35 -0
  166. package/dist/context/scan/relationship-benchmarks.js +5 -3
  167. package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
  168. package/dist/context/scan/relationship-composite-candidates.js +13 -1
  169. package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
  170. package/dist/context/scan/relationship-detection-budget.js +53 -0
  171. package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
  172. package/dist/context/scan/relationship-diagnostics.js +2 -0
  173. package/dist/context/scan/relationship-discovery.d.ts +9 -3
  174. package/dist/context/scan/relationship-discovery.js +27 -4
  175. package/dist/context/scan/relationship-llm-proposal.js +36 -27
  176. package/dist/context/scan/relationship-profiling.d.ts +7 -3
  177. package/dist/context/scan/relationship-profiling.js +53 -41
  178. package/dist/context/scan/relationship-validation.d.ts +6 -4
  179. package/dist/context/scan/relationship-validation.js +46 -32
  180. package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
  181. package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
  182. package/dist/context/scan/types.d.ts +5 -3
  183. package/dist/context/scan/types.js +2 -0
  184. package/dist/context/sl/local-query.js +5 -0
  185. package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
  186. package/dist/context/sl/semantic-layer.service.js +1 -1
  187. package/dist/context/sl/source-files.d.ts +5 -0
  188. package/dist/context/sl/source-files.js +17 -11
  189. package/dist/context/sl/tools/connection-id-schema.js +1 -1
  190. package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
  191. package/dist/context/sql-analysis/dialect-notes.js +40 -0
  192. package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
  193. package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
  194. package/dist/context/sql-analysis/dialects/mysql.md +9 -0
  195. package/dist/context/sql-analysis/dialects/postgres.md +10 -0
  196. package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
  197. package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
  198. package/dist/context/sql-analysis/dialects/tsql.md +10 -0
  199. package/dist/context/wiki/keys.js +1 -1
  200. package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
  201. package/dist/context/wiki/knowledge-wiki.service.js +2 -1
  202. package/dist/context/wiki/local-knowledge.d.ts +13 -0
  203. package/dist/context/wiki/local-knowledge.js +50 -6
  204. package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
  205. package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
  206. package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
  207. package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
  208. package/dist/context/wiki/types.d.ts +6 -0
  209. package/dist/context-build-view.d.ts +3 -0
  210. package/dist/context-build-view.js +1 -0
  211. package/dist/knowledge.d.ts +2 -0
  212. package/dist/knowledge.js +12 -1
  213. package/dist/local-adapters.js +11 -0
  214. package/dist/mcp-http-server.js +15 -1
  215. package/dist/mcp-server-factory.d.ts +2 -0
  216. package/dist/mcp-server-factory.js +15 -1
  217. package/dist/mcp-stdio-server.js +10 -2
  218. package/dist/notion-page-picker.js +1 -1
  219. package/dist/prompts/memory_agent_external_ingest.md +2 -0
  220. package/dist/public-ingest.d.ts +3 -1
  221. package/dist/public-ingest.js +3 -0
  222. package/dist/scan.d.ts +3 -1
  223. package/dist/scan.js +7 -0
  224. package/dist/setup-databases.d.ts +1 -1
  225. package/dist/setup-databases.js +43 -1
  226. package/dist/setup-sources.d.ts +5 -1
  227. package/dist/setup-sources.js +124 -48
  228. package/dist/setup.d.ts +3 -0
  229. package/dist/setup.js +6 -1
  230. package/dist/skills/analytics/SKILL.md +200 -2
  231. package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
  232. package/dist/skills/wiki_capture/SKILL.md +24 -0
  233. package/dist/sql.js +4 -0
  234. package/dist/status-project.d.ts +4 -0
  235. package/dist/status-project.js +54 -2
  236. package/dist/telemetry/events.d.ts +2 -2
  237. package/dist/text-ingest.d.ts +4 -0
  238. package/dist/text-ingest.js +58 -29
  239. package/dist/verbatim-ingest.d.ts +46 -0
  240. package/dist/verbatim-ingest.js +193 -0
  241. package/package.json +5 -1
  242. package/dist/context/ingest/final-gate-repair.d.ts +0 -28
  243. package/dist/context/ingest/final-gate-repair.js +0 -91
@@ -1,11 +1,13 @@
1
1
  import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js';
2
2
  import { type KtxScanRelationshipConfig } from '../project/config.js';
3
- import { type KtxScanEnrichmentStateStore } from './enrichment-state.js';
3
+ import { type KtxScanEmbeddingIdentity, type KtxScanEnrichmentStateStore, type KtxScanLlmIdentity } from './enrichment-state.js';
4
+ import type { KtxScanDescriptionResumeStore } from './local-enrichment-artifacts.js';
4
5
  import type { KtxEmbeddingUpdate, KtxEnrichedSchema, KtxRelationshipUpdate } from './enrichment-types.js';
5
6
  import type { KtxCompositeRelationshipCandidate } from './relationship-composite-candidates.js';
7
+ import type { KtxRelationshipDetectionStopReason } from './relationship-detection-budget.js';
6
8
  import type { KtxResolvedRelationshipDiscoveryCandidate } from './relationship-graph-resolver.js';
7
9
  import type { KtxRelationshipProfileArtifact } from './relationship-profiling.js';
8
- import type { KtxEmbeddingPort, KtxScanConnector, KtxScanContext, KtxScanEnrichmentStateSummary, KtxScanEnrichmentSummary, KtxScanMode, KtxScanRelationshipSummary, KtxScanWarning, KtxSchemaSnapshot, KtxTableRef } from './types.js';
10
+ import type { KtxEmbeddingPort, KtxScanConnector, KtxScanContext, KtxScanEnrichmentStage, KtxScanEnrichmentStateSummary, KtxScanEnrichmentSummary, KtxScanMode, KtxScanRelationshipSummary, KtxScanWarning, KtxSchemaSnapshot, KtxTableRef } from './types.js';
9
11
  export interface KtxLocalScanEnrichmentProviders {
10
12
  llmRuntime: KtxLlmRuntimePort;
11
13
  embedding?: KtxEmbeddingPort | null;
@@ -14,15 +16,45 @@ export interface KtxLocalScanEnrichmentInput {
14
16
  connectionId: string;
15
17
  mode: KtxScanMode;
16
18
  detectRelationships?: boolean;
19
+ /**
20
+ * Enrichment stages to (re)run this invocation. Undefined runs every eligible
21
+ * stage and respects the completed-stage short-circuit (spec-19 resume). When
22
+ * present, only the named stages run — each force-recomputes (bypassing the
23
+ * short-circuit) while unselected stages are left untouched on disk (D3).
24
+ */
25
+ stages?: KtxScanEnrichmentStage[];
17
26
  connector: KtxScanConnector;
18
27
  snapshot?: KtxSchemaSnapshot;
19
28
  context: KtxScanContext;
20
29
  providers: KtxLocalScanEnrichmentProviders | null;
21
30
  stateStore?: KtxScanEnrichmentStateStore | null;
31
+ /**
32
+ * Durable per-batch resume record for the descriptions stage. When present, an
33
+ * interrupted descriptions stage resumes by re-enriching only the tables not
34
+ * already flushed (inputHash-gated). Null/undefined disables incremental flush.
35
+ */
36
+ descriptionResumeStore?: KtxScanDescriptionResumeStore | null;
37
+ /**
38
+ * Lazily loads the descriptions already persisted in the on-disk _schema, used
39
+ * to feed embeddings + relationships their description context when the
40
+ * descriptions stage does not run this invocation (e.g. `--stages relationships`).
41
+ * Called at most once and only when a downstream stage needs it, so a normal
42
+ * full run never pays the read.
43
+ */
44
+ loadPriorDescriptions?: (snapshot: KtxSchemaSnapshot) => Promise<KtxLocalScanEnrichmentResult['descriptionUpdates']>;
22
45
  syncId?: string;
23
- providerIdentity?: Record<string, unknown>;
46
+ /** Description-LLM identity that keys the descriptions + relationships stage hashes. */
47
+ llmIdentity?: KtxScanLlmIdentity;
48
+ /** Embedding-model identity that keys the embeddings stage hash. */
49
+ embeddingIdentity?: KtxScanEmbeddingIdentity;
24
50
  relationshipSettings?: KtxScanRelationshipConfig;
25
51
  now?: () => Date;
52
+ /**
53
+ * Invoked once the last non-relationship stage completes and before
54
+ * relationship detection runs, so the descriptions + embeddings reach the
55
+ * queryable layer even if the relationship stage is later interrupted.
56
+ */
57
+ onCheckpoint?: (checkpoint: KtxLocalScanEnrichmentResult) => Promise<void>;
26
58
  }
27
59
  export interface KtxLocalScanEnrichmentResult {
28
60
  snapshot: KtxSchemaSnapshot;
@@ -40,7 +72,10 @@ export interface KtxLocalScanEnrichmentResult {
40
72
  relationshipProfile: KtxRelationshipProfileArtifact | null;
41
73
  resolvedRelationships: KtxResolvedRelationshipDiscoveryCandidate[] | null;
42
74
  compositeRelationships: KtxCompositeRelationshipCandidate[] | null;
75
+ relationshipPartial: {
76
+ reason: KtxRelationshipDetectionStopReason;
77
+ } | null;
43
78
  }
44
79
  export declare function createDeterministicLocalScanEnrichmentProviders(): KtxLocalScanEnrichmentProviders;
45
- export declare function snapshotToKtxEnrichedSchema(snapshot: KtxSchemaSnapshot, embeddingsByColumnId?: ReadonlyMap<string, number[]>): KtxEnrichedSchema;
80
+ export declare function snapshotToKtxEnrichedSchema(snapshot: KtxSchemaSnapshot, embeddingsByColumnId?: ReadonlyMap<string, number[]>, descriptions?: KtxLocalScanEnrichmentResult['descriptionUpdates']): KtxEnrichedSchema;
46
81
  export declare function runLocalScanEnrichment(input: KtxLocalScanEnrichmentInput): Promise<KtxLocalScanEnrichmentResult>;