@kaelio/ktx 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/assets/python/{kaelio_ktx-0.13.0-py3-none-any.whl → kaelio_ktx-0.14.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-program.js +1 -1
  5. package/dist/commands/ingest-commands.d.ts +9 -0
  6. package/dist/commands/ingest-commands.js +37 -1
  7. package/dist/commands/knowledge-commands.js +3 -0
  8. package/dist/commands/setup-commands.js +14 -1
  9. package/dist/connection-drivers.d.ts +2 -0
  10. package/dist/connection-drivers.js +7 -3
  11. package/dist/connection.d.ts +3 -0
  12. package/dist/connection.js +27 -0
  13. package/dist/connectors/bigquery/connector.d.ts +17 -6
  14. package/dist/connectors/bigquery/connector.js +152 -76
  15. package/dist/connectors/bigquery/dialect.d.ts +2 -2
  16. package/dist/connectors/clickhouse/connector.d.ts +1 -0
  17. package/dist/connectors/clickhouse/connector.js +45 -16
  18. package/dist/connectors/clickhouse/dialect.d.ts +2 -2
  19. package/dist/connectors/mongodb/connector.d.ts +69 -0
  20. package/dist/connectors/mongodb/connector.js +307 -0
  21. package/dist/connectors/mongodb/dialect.d.ts +17 -0
  22. package/dist/connectors/mongodb/dialect.js +50 -0
  23. package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
  24. package/dist/connectors/mongodb/live-database-introspection.js +24 -0
  25. package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
  26. package/dist/connectors/mongodb/schema-inference.js +110 -0
  27. package/dist/connectors/mysql/connector.d.ts +1 -0
  28. package/dist/connectors/mysql/connector.js +18 -2
  29. package/dist/connectors/mysql/dialect.d.ts +2 -2
  30. package/dist/connectors/postgres/connector.d.ts +1 -0
  31. package/dist/connectors/postgres/connector.js +20 -3
  32. package/dist/connectors/postgres/dialect.d.ts +2 -2
  33. package/dist/connectors/snowflake/connector.d.ts +1 -0
  34. package/dist/connectors/snowflake/connector.js +27 -3
  35. package/dist/connectors/snowflake/dialect.d.ts +2 -2
  36. package/dist/connectors/sqlite/connector.d.ts +11 -1
  37. package/dist/connectors/sqlite/connector.js +120 -17
  38. package/dist/connectors/sqlite/dialect.d.ts +2 -2
  39. package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
  40. package/dist/connectors/sqlite/read-query-child.js +23 -0
  41. package/dist/connectors/sqlserver/connector.d.ts +2 -0
  42. package/dist/connectors/sqlserver/connector.js +23 -5
  43. package/dist/connectors/sqlserver/dialect.d.ts +2 -2
  44. package/dist/context/cache/content-result-cache.d.ts +36 -0
  45. package/dist/context/cache/content-result-cache.js +14 -0
  46. package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
  47. package/dist/context/cache/sqlite-content-result-cache.js +223 -0
  48. package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
  49. package/dist/context/connections/bigquery-identifiers.js +7 -0
  50. package/dist/context/connections/configured-connections.d.ts +9 -0
  51. package/dist/context/connections/configured-connections.js +18 -0
  52. package/dist/context/connections/dialects.d.ts +30 -4
  53. package/dist/context/connections/dialects.js +38 -11
  54. package/dist/context/connections/drivers.js +21 -0
  55. package/dist/context/connections/gdrive-config.d.ts +19 -0
  56. package/dist/context/connections/gdrive-config.js +57 -0
  57. package/dist/context/connections/query-deadline.d.ts +31 -0
  58. package/dist/context/connections/query-deadline.js +37 -0
  59. package/dist/context/connections/read-only-sql.d.ts +5 -0
  60. package/dist/context/connections/read-only-sql.js +139 -1
  61. package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
  62. package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
  63. package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
  64. package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
  65. package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
  66. package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
  67. package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
  68. package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
  69. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
  70. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
  71. package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
  72. package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
  73. package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
  74. package/dist/context/ingest/adapters/gdrive/types.js +36 -0
  75. package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
  76. package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
  77. package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
  78. package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
  79. package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
  80. package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
  81. package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
  82. package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
  83. package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
  84. package/dist/context/ingest/adapters/live-database/stage.js +5 -5
  85. package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
  86. package/dist/context/ingest/adapters/metabase/types.js +1 -1
  87. package/dist/context/ingest/artifact-gates.d.ts +32 -1
  88. package/dist/context/ingest/artifact-gates.js +85 -18
  89. package/dist/context/ingest/final-gate-prune.d.ts +35 -0
  90. package/dist/context/ingest/final-gate-prune.js +229 -0
  91. package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
  92. package/dist/context/ingest/ingest-bundle.runner.js +459 -240
  93. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
  94. package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
  95. package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
  96. package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
  97. package/dist/context/ingest/local-adapters.js +6 -0
  98. package/dist/context/ingest/local-bundle-runtime.js +7 -2
  99. package/dist/context/ingest/local-ingest.js +1 -1
  100. package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
  101. package/dist/context/ingest/local-stage-ingest.js +3 -1
  102. package/dist/context/ingest/ports.d.ts +3 -0
  103. package/dist/context/ingest/report-snapshot.js +13 -3
  104. package/dist/context/ingest/reports.d.ts +3 -3
  105. package/dist/context/ingest/reports.js +3 -1
  106. package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
  107. package/dist/context/ingest/stages/build-wu-context.js +2 -1
  108. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
  109. package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
  110. package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
  111. package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
  112. package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
  113. package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
  114. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
  115. package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
  116. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
  117. package/dist/context/ingest/types.d.ts +3 -0
  118. package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
  119. package/dist/context/ingest/wiki-body-refs.js +37 -8
  120. package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
  121. package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
  122. package/dist/context/ingest/work-unit-cache.d.ts +67 -0
  123. package/dist/context/ingest/work-unit-cache.js +230 -0
  124. package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
  125. package/dist/context/llm/ai-sdk-runtime.js +5 -0
  126. package/dist/context/llm/claude-code-runtime.d.ts +4 -1
  127. package/dist/context/llm/claude-code-runtime.js +38 -8
  128. package/dist/context/llm/codex-runtime.d.ts +4 -1
  129. package/dist/context/llm/codex-runtime.js +48 -35
  130. package/dist/context/llm/runtime-port.d.ts +26 -0
  131. package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
  132. package/dist/context/llm/subprocess-generate-object-child.js +34 -0
  133. package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
  134. package/dist/context/llm/subprocess-generate-object.js +122 -0
  135. package/dist/context/mcp/context-tools.d.ts +2 -0
  136. package/dist/context/mcp/context-tools.js +111 -17
  137. package/dist/context/mcp/local-project-ports.d.ts +7 -0
  138. package/dist/context/mcp/local-project-ports.js +29 -0
  139. package/dist/context/mcp/logger.d.ts +24 -0
  140. package/dist/context/mcp/logger.js +49 -0
  141. package/dist/context/mcp/server.js +2 -0
  142. package/dist/context/mcp/types.d.ts +17 -0
  143. package/dist/context/memory/memory-agent.service.js +1 -1
  144. package/dist/context/project/config.d.ts +42 -0
  145. package/dist/context/project/config.js +5 -0
  146. package/dist/context/project/driver-schemas.d.ts +20 -0
  147. package/dist/context/project/driver-schemas.js +50 -1
  148. package/dist/context/project/project.js +1 -1
  149. package/dist/context/project/setup-config.js +1 -0
  150. package/dist/context/scan/description-generation.d.ts +4 -0
  151. package/dist/context/scan/description-generation.js +100 -13
  152. package/dist/context/scan/enabled-tables.d.ts +5 -0
  153. package/dist/context/scan/enabled-tables.js +13 -1
  154. package/dist/context/scan/enrichment-state.d.ts +50 -7
  155. package/dist/context/scan/enrichment-state.js +30 -13
  156. package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
  157. package/dist/context/scan/local-enrichment-artifacts.js +155 -32
  158. package/dist/context/scan/local-enrichment.d.ts +39 -4
  159. package/dist/context/scan/local-enrichment.js +345 -92
  160. package/dist/context/scan/local-scan.d.ts +4 -1
  161. package/dist/context/scan/local-scan.js +54 -13
  162. package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
  163. package/dist/context/scan/local-structural-artifacts.js +5 -0
  164. package/dist/context/scan/object-introspection.d.ts +21 -0
  165. package/dist/context/scan/object-introspection.js +35 -0
  166. package/dist/context/scan/relationship-benchmarks.js +5 -3
  167. package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
  168. package/dist/context/scan/relationship-composite-candidates.js +13 -1
  169. package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
  170. package/dist/context/scan/relationship-detection-budget.js +53 -0
  171. package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
  172. package/dist/context/scan/relationship-diagnostics.js +2 -0
  173. package/dist/context/scan/relationship-discovery.d.ts +9 -3
  174. package/dist/context/scan/relationship-discovery.js +27 -4
  175. package/dist/context/scan/relationship-llm-proposal.js +36 -27
  176. package/dist/context/scan/relationship-profiling.d.ts +7 -3
  177. package/dist/context/scan/relationship-profiling.js +53 -41
  178. package/dist/context/scan/relationship-validation.d.ts +6 -4
  179. package/dist/context/scan/relationship-validation.js +46 -32
  180. package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
  181. package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
  182. package/dist/context/scan/types.d.ts +5 -3
  183. package/dist/context/scan/types.js +2 -0
  184. package/dist/context/sl/local-query.js +5 -0
  185. package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
  186. package/dist/context/sl/semantic-layer.service.js +1 -1
  187. package/dist/context/sl/source-files.d.ts +5 -0
  188. package/dist/context/sl/source-files.js +17 -11
  189. package/dist/context/sl/tools/connection-id-schema.js +1 -1
  190. package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
  191. package/dist/context/sql-analysis/dialect-notes.js +40 -0
  192. package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
  193. package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
  194. package/dist/context/sql-analysis/dialects/mysql.md +9 -0
  195. package/dist/context/sql-analysis/dialects/postgres.md +10 -0
  196. package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
  197. package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
  198. package/dist/context/sql-analysis/dialects/tsql.md +10 -0
  199. package/dist/context/wiki/keys.js +1 -1
  200. package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
  201. package/dist/context/wiki/knowledge-wiki.service.js +2 -1
  202. package/dist/context/wiki/local-knowledge.d.ts +13 -0
  203. package/dist/context/wiki/local-knowledge.js +50 -6
  204. package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
  205. package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
  206. package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
  207. package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
  208. package/dist/context/wiki/types.d.ts +6 -0
  209. package/dist/context-build-view.d.ts +3 -0
  210. package/dist/context-build-view.js +1 -0
  211. package/dist/knowledge.d.ts +2 -0
  212. package/dist/knowledge.js +12 -1
  213. package/dist/local-adapters.js +11 -0
  214. package/dist/mcp-http-server.js +15 -1
  215. package/dist/mcp-server-factory.d.ts +2 -0
  216. package/dist/mcp-server-factory.js +15 -1
  217. package/dist/mcp-stdio-server.js +10 -2
  218. package/dist/notion-page-picker.js +1 -1
  219. package/dist/prompts/memory_agent_external_ingest.md +2 -0
  220. package/dist/public-ingest.d.ts +3 -1
  221. package/dist/public-ingest.js +3 -0
  222. package/dist/scan.d.ts +3 -1
  223. package/dist/scan.js +7 -0
  224. package/dist/setup-databases.d.ts +1 -1
  225. package/dist/setup-databases.js +43 -1
  226. package/dist/setup-sources.d.ts +5 -1
  227. package/dist/setup-sources.js +124 -48
  228. package/dist/setup.d.ts +3 -0
  229. package/dist/setup.js +6 -1
  230. package/dist/skills/analytics/SKILL.md +200 -2
  231. package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
  232. package/dist/skills/wiki_capture/SKILL.md +24 -0
  233. package/dist/sql.js +4 -0
  234. package/dist/status-project.d.ts +4 -0
  235. package/dist/status-project.js +54 -2
  236. package/dist/telemetry/events.d.ts +2 -2
  237. package/dist/text-ingest.d.ts +4 -0
  238. package/dist/text-ingest.js +58 -29
  239. package/dist/verbatim-ingest.d.ts +46 -0
  240. package/dist/verbatim-ingest.js +193 -0
  241. package/package.json +5 -1
  242. package/dist/context/ingest/final-gate-repair.d.ts +0 -28
  243. package/dist/context/ingest/final-gate-repair.js +0 -91
@@ -0,0 +1,40 @@
1
+ import { KtxExpectedError } from '../../../../errors.js';
2
+ import { tableRefFromKey } from '../../../scan/table-ref.js';
3
+ const OBJECT_SKIP_CODE = 'object_introspection_failed';
4
+ function formatScopeEntry(key) {
5
+ const ref = tableRefFromKey(key);
6
+ return [ref.catalog, ref.db, ref.name].filter((part) => Boolean(part)).join('.');
7
+ }
8
+ function discoveredObjectNames(snapshot) {
9
+ const raw = snapshot.metadata.discovered_object_names;
10
+ return Array.isArray(raw) ? raw.filter((value) => typeof value === 'string') : [];
11
+ }
12
+ /**
13
+ * Enforces the partial-vs-total outcome rules for a live-database snapshot,
14
+ * uniformly for every connector. Outcomes follow from object counts, not a
15
+ * mode: a connection with at least one ingested object succeeds (any broken
16
+ * objects ride along as warnings); a connection where every introspected object
17
+ * failed, or a non-empty enabled_tables scope that matched nothing, raises a
18
+ * clear connection error instead of staging an empty layer that would later
19
+ * surface as the generic "did not recognize" message. A legitimately empty
20
+ * database (no scope, no objects) succeeds with an empty layer.
21
+ */
22
+ export function assertLiveDatabaseScanOutcome(input) {
23
+ const { connectionId, scope, snapshot } = input;
24
+ if (snapshot.tables.length > 0) {
25
+ return;
26
+ }
27
+ const skipped = (snapshot.warnings ?? []).filter((warning) => warning.code === OBJECT_SKIP_CODE);
28
+ if (skipped.length > 0) {
29
+ const detail = skipped.map((warning) => `${warning.table ?? 'object'}: ${warning.message}`).join('; ');
30
+ throw new KtxExpectedError(`Connection "${connectionId}" produced no semantic layer: all ${skipped.length} introspected ` +
31
+ `${skipped.length === 1 ? 'object' : 'objects'} failed (${detail}).`);
32
+ }
33
+ if (scope && scope.size > 0) {
34
+ const requested = [...scope].map(formatScopeEntry).sort();
35
+ const available = discoveredObjectNames(snapshot);
36
+ const availableClause = available.length > 0 ? ` Available objects: ${available.join(', ')}.` : '';
37
+ throw new KtxExpectedError(`enabled_tables for connection "${connectionId}" matched no objects ` +
38
+ `(looked for: ${requested.join(', ')}).${availableClause}`);
39
+ }
40
+ }
@@ -101,13 +101,13 @@ export async function readLiveDatabaseTableFiles(stagedDir) {
101
101
  return out;
102
102
  }
103
103
  export async function detectLiveDatabaseStagedDir(stagedDir) {
104
+ // A valid live-database staging is identified by its connection.json marker.
105
+ // An empty table set is a legitimate outcome (an empty database), so the
106
+ // presence of table files is not required — the total-vs-partial decision is
107
+ // made earlier by assertLiveDatabaseScanOutcome, before staging.
104
108
  try {
105
109
  const meta = JSON.parse(await readFile(join(stagedDir, LIVE_DATABASE_META_FILE), 'utf8'));
106
- if (!meta || typeof meta !== 'object' || Array.isArray(meta)) {
107
- return false;
108
- }
109
- const files = await readLiveDatabaseTableFiles(stagedDir);
110
- return files.length > 0;
110
+ return Boolean(meta) && typeof meta === 'object' && !Array.isArray(meta);
111
111
  }
112
112
  catch {
113
113
  return false;
@@ -118,8 +118,8 @@ export declare const stagedSyncConfigSchema: z.ZodObject<{
118
118
  }>;
119
119
  selections: z.ZodArray<z.ZodObject<{
120
120
  selectionType: z.ZodEnum<{
121
- collection: "collection";
122
121
  item: "item";
122
+ collection: "collection";
123
123
  }>;
124
124
  metabaseObjectId: z.ZodNumber;
125
125
  }, z.core.$strip>>;
@@ -1,6 +1,6 @@
1
1
  import { z } from 'zod';
2
2
  const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']);
3
- const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/);
3
+ const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/);
4
4
  /**
5
5
  * The lean config the adapter needs at `fetch()` time. Lives in the BullMQ payload's
6
6
  * `bundleRef.config` when the runner invokes the adapter. Never persisted beyond one
@@ -2,6 +2,7 @@ import type { SemanticLayerService } from '../../context/sl/semantic-layer.servi
2
2
  import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
3
3
  import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
4
4
  import type { WuValidationResult } from './stages/validate-wu-sources.js';
5
+ import { type WikiBodyRefIssue } from './wiki-body-refs.js';
5
6
  export interface FinalArtifactGateInput {
6
7
  connectionIds: string[];
7
8
  changedWikiPageKeys: string[];
@@ -18,5 +19,35 @@ export interface ProvenanceRawPathValidationInput {
18
19
  currentRawPaths: Set<string>;
19
20
  deletedRawPaths: Set<string>;
20
21
  }
21
- export declare function validateFinalIngestArtifacts(input: FinalArtifactGateInput): Promise<void>;
22
+ export type FinalArtifactGateFinding = {
23
+ kind: 'invalid_source';
24
+ connectionId: string;
25
+ sourceName: string;
26
+ errors: string[];
27
+ } | {
28
+ kind: 'missing_join_target';
29
+ ownerConnectionId: string;
30
+ ownerSourceName: string;
31
+ targetSourceName: string;
32
+ message: string;
33
+ } | {
34
+ kind: 'missing_wiki_ref';
35
+ pageKey: string;
36
+ targetPageKey: string;
37
+ message: string;
38
+ } | {
39
+ kind: 'missing_wiki_sl_ref';
40
+ pageKey: string;
41
+ ref: string;
42
+ sourceName: string;
43
+ entityName: string | null;
44
+ message: string;
45
+ } | WikiBodyRefIssue;
46
+ export interface FinalArtifactGateResult {
47
+ ok: boolean;
48
+ findings: FinalArtifactGateFinding[];
49
+ }
50
+ export declare function formatFinalArtifactGateFindings(findings: FinalArtifactGateFinding[]): string;
51
+ export declare function isFinalArtifactGateFindingPruneable(finding: FinalArtifactGateFinding): boolean;
52
+ export declare function validateFinalIngestArtifacts(input: FinalArtifactGateInput): Promise<FinalArtifactGateResult>;
22
53
  export declare function validateProvenanceRawPaths(input: ProvenanceRawPathValidationInput): void;
@@ -1,5 +1,8 @@
1
1
  import { findMissingWikiRefs } from '../wiki/wiki-ref-validation.js';
2
- import { findInvalidWikiBodyRefs } from './wiki-body-refs.js';
2
+ import { findInvalidWikiBodyRefIssues } from './wiki-body-refs.js';
3
+ function normalizeRawPath(path) {
4
+ return path.replace(/\\/g, '/').replace(/^\/+/, '');
5
+ }
3
6
  function parseSlRef(ref) {
4
7
  const withoutConnection = ref.includes('/') ? ref.slice(ref.indexOf('/') + 1) : ref;
5
8
  const connectionId = ref.includes('/') ? ref.slice(0, ref.indexOf('/')) : null;
@@ -14,7 +17,7 @@ function slEntityNames(source) {
14
17
  ]);
15
18
  }
16
19
  async function validateWikiSlRefs(input) {
17
- const errors = [];
20
+ const findings = [];
18
21
  const sourcesByConnection = new Map();
19
22
  for (const connectionId of input.connectionIds) {
20
23
  const { sources } = await input.semanticLayerService.loadAllSources(connectionId);
@@ -36,18 +39,32 @@ async function validateWikiSlRefs(input) {
36
39
  }
37
40
  }
38
41
  if (!source) {
39
- errors.push(`${pageKey}: unknown sl_refs entry ${ref}`);
42
+ findings.push({
43
+ kind: 'missing_wiki_sl_ref',
44
+ pageKey,
45
+ ref,
46
+ sourceName: parsed.sourceName,
47
+ entityName: parsed.entityName,
48
+ message: `${pageKey}: unknown sl_refs entry ${ref}`,
49
+ });
40
50
  continue;
41
51
  }
42
52
  if (parsed.entityName && !slEntityNames(source).has(parsed.entityName)) {
43
- errors.push(`${pageKey}: unknown sl_refs entity ${ref}`);
53
+ findings.push({
54
+ kind: 'missing_wiki_sl_ref',
55
+ pageKey,
56
+ ref,
57
+ sourceName: parsed.sourceName,
58
+ entityName: parsed.entityName,
59
+ message: `${pageKey}: unknown sl_refs entity ${ref}`,
60
+ });
44
61
  }
45
62
  }
46
63
  }
47
- return errors;
64
+ return findings;
48
65
  }
49
66
  async function validateWikiRefs(input) {
50
- const dangling = [];
67
+ const findings = [];
51
68
  for (const pageKey of input.changedWikiPageKeys) {
52
69
  const page = await input.wikiService.readPage('GLOBAL', null, pageKey);
53
70
  if (!page) {
@@ -62,28 +79,77 @@ async function validateWikiRefs(input) {
62
79
  content: page.content,
63
80
  });
64
81
  for (const missingRef of missingRefs) {
65
- dangling.push(`${pageKey} -> ${missingRef}`);
82
+ findings.push({
83
+ kind: 'missing_wiki_ref',
84
+ pageKey,
85
+ targetPageKey: missingRef,
86
+ message: `${pageKey} -> ${missingRef}`,
87
+ });
88
+ }
89
+ }
90
+ return findings;
91
+ }
92
+ export function formatFinalArtifactGateFindings(findings) {
93
+ const errors = findings.map((finding) => {
94
+ if (finding.kind === 'invalid_source') {
95
+ return `semantic-layer validation failed for ${finding.connectionId}:${finding.sourceName}: ${finding.errors.join('; ')}`;
96
+ }
97
+ if (finding.kind === 'missing_wiki_ref') {
98
+ return `wiki reference targets missing page: ${finding.message}`;
99
+ }
100
+ return finding.message;
101
+ });
102
+ return `final artifact gates failed:\n${errors.join('\n')}`;
103
+ }
104
+ export function isFinalArtifactGateFindingPruneable(finding) {
105
+ switch (finding.kind) {
106
+ case 'invalid_source':
107
+ case 'missing_join_target':
108
+ case 'missing_wiki_ref':
109
+ case 'missing_wiki_sl_ref':
110
+ case 'missing_wiki_body_sl_entity':
111
+ case 'missing_wiki_body_sl_source':
112
+ case 'missing_wiki_body_table':
113
+ return true;
114
+ default: {
115
+ const exhaustive = finding;
116
+ return exhaustive;
66
117
  }
67
118
  }
68
- return dangling;
69
119
  }
70
120
  export async function validateFinalIngestArtifacts(input) {
71
121
  // Join-neighbor expansion happens inside validateTouchedSources so work-unit
72
122
  // validation and this gate check the same set — a source that passes one
73
123
  // passes the other.
74
124
  const validation = await input.validateTouchedSources(input.touchedSlSources);
75
- const errors = validation.invalidSources.map((invalid) => `semantic-layer validation failed for ${invalid.source}: ${invalid.errors.join('; ')}`);
76
- errors.push(...(await validateWikiSlRefs(input)));
77
- const danglingWikiRefs = await validateWikiRefs(input);
78
- if (danglingWikiRefs.length > 0) {
79
- errors.push(`wiki references target missing page(s): ${danglingWikiRefs.join(', ')}`);
125
+ const findings = [];
126
+ for (const invalid of validation.invalidSources) {
127
+ const [connectionId = '', sourceName = ''] = invalid.source.split(':', 2);
128
+ const issues = invalid.issues ?? invalid.errors.map((message) => ({ kind: 'source_validation', message }));
129
+ const sourceErrors = issues.filter((issue) => issue.kind === 'source_validation').map((issue) => issue.message);
130
+ if (sourceErrors.length > 0) {
131
+ findings.push({ kind: 'invalid_source', connectionId, sourceName, errors: sourceErrors });
132
+ }
133
+ for (const issue of issues) {
134
+ if (issue.kind === 'missing_join_target') {
135
+ findings.push({
136
+ kind: 'missing_join_target',
137
+ ownerConnectionId: connectionId,
138
+ ownerSourceName: sourceName,
139
+ targetSourceName: issue.targetSourceName,
140
+ message: issue.message,
141
+ });
142
+ }
143
+ }
80
144
  }
145
+ findings.push(...(await validateWikiSlRefs(input)));
146
+ findings.push(...(await validateWikiRefs(input)));
81
147
  for (const pageKey of input.changedWikiPageKeys) {
82
148
  const page = await input.wikiService.readPage('GLOBAL', null, pageKey);
83
149
  if (!page) {
84
150
  continue;
85
151
  }
86
- errors.push(...(await findInvalidWikiBodyRefs({
152
+ findings.push(...(await findInvalidWikiBodyRefIssues({
87
153
  pageKey,
88
154
  body: page.content,
89
155
  visibleConnectionIds: input.connectionIds,
@@ -94,13 +160,14 @@ export async function validateFinalIngestArtifacts(input) {
94
160
  tableExists: input.tableExists,
95
161
  })));
96
162
  }
97
- if (errors.length > 0) {
98
- throw new Error(`final artifact gates failed:\n${errors.join('\n')}`);
99
- }
163
+ return { ok: findings.length === 0, findings };
100
164
  }
101
165
  export function validateProvenanceRawPaths(input) {
166
+ const currentRawPaths = new Set([...input.currentRawPaths].map(normalizeRawPath));
167
+ const deletedRawPaths = new Set([...input.deletedRawPaths].map(normalizeRawPath));
102
168
  for (const row of input.rows) {
103
- if (!input.currentRawPaths.has(row.rawPath) && !input.deletedRawPaths.has(row.rawPath)) {
169
+ const rawPath = normalizeRawPath(row.rawPath);
170
+ if (!currentRawPaths.has(rawPath) && !deletedRawPaths.has(rawPath)) {
104
171
  throw new Error(`provenance row references raw path outside this snapshot: ${row.rawPath}`);
105
172
  }
106
173
  }
@@ -0,0 +1,35 @@
1
+ import type { KtxFileStorePort } from '../core/file-store.js';
2
+ import type { KnowledgeWikiService } from '../wiki/knowledge-wiki.service.js';
3
+ import type { FinalArtifactGateFinding } from './artifact-gates.js';
4
+ import type { IngestTraceWriter } from './ingest-trace.js';
5
+ type FinalGatePrunedReferenceKind = 'join' | 'wiki_ref' | 'wiki_sl_ref' | 'wiki_body_ref';
6
+ type SemanticLayerFileStore = Pick<KtxFileStorePort, 'readFile' | 'writeFile' | 'deleteFile' | 'listFiles'>;
7
+ export interface FinalGatePrunedReference {
8
+ kind: FinalGatePrunedReferenceKind;
9
+ artifact: string;
10
+ removedRef: string;
11
+ absentTarget: string;
12
+ }
13
+ export interface FinalGateDroppedSource {
14
+ connectionId: string;
15
+ sourceName: string;
16
+ reason: string;
17
+ }
18
+ export interface FinalGatePruneResult {
19
+ prunedReferences: FinalGatePrunedReference[];
20
+ droppedSources: FinalGateDroppedSource[];
21
+ }
22
+ interface PruneInput {
23
+ workdir: string;
24
+ semanticLayerFiles: SemanticLayerFileStore;
25
+ findings: FinalArtifactGateFinding[];
26
+ droppedSources: FinalGateDroppedSource[];
27
+ trace: IngestTraceWriter;
28
+ author: {
29
+ name: string;
30
+ email: string;
31
+ };
32
+ wikiService?: KnowledgeWikiService;
33
+ }
34
+ export declare function pruneFinalGateFindings(input: PruneInput): Promise<FinalGatePruneResult>;
35
+ export {};
@@ -0,0 +1,229 @@
1
+ import YAML from 'yaml';
2
+ import { listSlSourceFiles, resolveSlSourceFile, slSourceNameForFile } from '../sl/source-files.js';
3
+ async function resolveYamlSource(fileStore, connectionId, sourceName) {
4
+ const file = await resolveSlSourceFile(fileStore, connectionId, sourceName);
5
+ if (!file) {
6
+ return null;
7
+ }
8
+ const parsed = YAML.parse(file.content);
9
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
10
+ throw new Error(`${file.path}: expected semantic-layer source YAML object`);
11
+ }
12
+ return { path: file.path, source: parsed };
13
+ }
14
+ async function writeYamlSource(input) {
15
+ await input.fileStore.writeFile(input.path, YAML.stringify(input.source, { indent: 2, lineWidth: 0, version: '1.1' }), input.author.name, input.author.email, `Prune dangling joins from ${input.path}`, { skipLock: true });
16
+ }
17
+ function removeInlineToken(content, rawToken) {
18
+ return content.replaceAll(`\`${rawToken}\``, '').replace(/[ \t]+([.,;:!?])/g, '$1');
19
+ }
20
+ function escapeRegExp(value) {
21
+ return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
22
+ }
23
+ function removeWikiRefToken(content, targetPageKey) {
24
+ const pattern = new RegExp(`\\[\\[\\s*${escapeRegExp(targetPageKey)}(?:\\|[^\\]\\n]+)?\\s*\\]\\]`, 'g');
25
+ return content.replace(pattern, '').replace(/[ \t]+([.,;:!?])/g, '$1');
26
+ }
27
+ function wikiBodyAbsentTarget(finding) {
28
+ if (finding.kind === 'missing_wiki_body_table') {
29
+ return finding.tableRef;
30
+ }
31
+ if (finding.kind === 'missing_wiki_body_sl_source') {
32
+ return finding.sourceName;
33
+ }
34
+ if (finding.kind === 'missing_wiki_body_sl_entity') {
35
+ return `${finding.sourceName}.${finding.entityName}`;
36
+ }
37
+ return '';
38
+ }
39
+ /** Remove every join whose target matches `shouldRemove`, write the file back, and
40
+ * emit one pruned-reference record per distinct removed target. */
41
+ async function pruneJoinsFromSource(input) {
42
+ if (!Array.isArray(input.resolved.source.joins)) {
43
+ return [];
44
+ }
45
+ const removed = new Set();
46
+ const nextJoins = input.resolved.source.joins.filter((entry) => {
47
+ const to = entry && typeof entry === 'object' && 'to' in entry ? entry.to : undefined;
48
+ if (typeof to === 'string' && input.shouldRemove(to)) {
49
+ removed.add(to);
50
+ return false;
51
+ }
52
+ return true;
53
+ });
54
+ if (removed.size === 0) {
55
+ return [];
56
+ }
57
+ input.resolved.source.joins = nextJoins;
58
+ await writeYamlSource({
59
+ fileStore: input.fileStore,
60
+ path: input.resolved.path,
61
+ source: input.resolved.source,
62
+ author: input.author,
63
+ });
64
+ const records = [];
65
+ for (const target of removed) {
66
+ const record = {
67
+ kind: 'join',
68
+ artifact: `semantic-layer/${input.connectionId}/${input.ownerSourceName}`,
69
+ removedRef: target,
70
+ absentTarget: target,
71
+ };
72
+ records.push(record);
73
+ await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
74
+ }
75
+ return records;
76
+ }
77
+ export async function pruneFinalGateFindings(input) {
78
+ const droppedSources = [...input.droppedSources];
79
+ const prunedReferences = [];
80
+ const droppedKey = new Set(droppedSources.map((source) => `${source.connectionId}:${source.sourceName}`));
81
+ for (const finding of input.findings) {
82
+ if (finding.kind !== 'invalid_source') {
83
+ continue;
84
+ }
85
+ const key = `${finding.connectionId}:${finding.sourceName}`;
86
+ if (droppedKey.has(key)) {
87
+ continue;
88
+ }
89
+ const file = await resolveSlSourceFile(input.semanticLayerFiles, finding.connectionId, finding.sourceName);
90
+ if (!file) {
91
+ continue;
92
+ }
93
+ const deleted = await input.semanticLayerFiles.deleteFile(file.path, input.author.name, input.author.email, `Drop invalid source ${finding.connectionId}:${finding.sourceName}`, { skipLock: true });
94
+ if (!deleted) {
95
+ continue;
96
+ }
97
+ const dropped = {
98
+ connectionId: finding.connectionId,
99
+ sourceName: finding.sourceName,
100
+ reason: finding.errors.join('; '),
101
+ };
102
+ droppedSources.push(dropped);
103
+ droppedKey.add(key);
104
+ await input.trace.event('info', 'final_gates', 'final_gate_source_dropped', dropped);
105
+ }
106
+ // A dropped node can leave a join dangling on any owner — including sources
107
+ // untouched by this run, which the touched-scoped gate (and the confirm gate
108
+ // after it) never revisit. Prune those edges directly (D5), or the committed
109
+ // orphan join breaks every SL query on the connection.
110
+ const droppedByConnection = new Map();
111
+ for (const dropped of droppedSources) {
112
+ const names = droppedByConnection.get(dropped.connectionId) ?? new Set();
113
+ names.add(dropped.sourceName);
114
+ droppedByConnection.set(dropped.connectionId, names);
115
+ }
116
+ for (const [connectionId, droppedNames] of droppedByConnection) {
117
+ for (const file of await listSlSourceFiles(input.semanticLayerFiles, connectionId)) {
118
+ let parsed;
119
+ try {
120
+ parsed = YAML.parse(file.content);
121
+ }
122
+ catch {
123
+ continue;
124
+ }
125
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
126
+ continue;
127
+ }
128
+ prunedReferences.push(...(await pruneJoinsFromSource({
129
+ fileStore: input.semanticLayerFiles,
130
+ connectionId,
131
+ ownerSourceName: slSourceNameForFile(file.path, file.content),
132
+ resolved: { path: file.path, source: parsed },
133
+ shouldRemove: (target) => droppedNames.has(target),
134
+ author: input.author,
135
+ trace: input.trace,
136
+ })));
137
+ }
138
+ }
139
+ for (const finding of input.findings) {
140
+ if (finding.kind !== 'missing_join_target') {
141
+ continue;
142
+ }
143
+ const resolved = await resolveYamlSource(input.semanticLayerFiles, finding.ownerConnectionId, finding.ownerSourceName);
144
+ if (!resolved) {
145
+ continue;
146
+ }
147
+ prunedReferences.push(...(await pruneJoinsFromSource({
148
+ fileStore: input.semanticLayerFiles,
149
+ connectionId: finding.ownerConnectionId,
150
+ ownerSourceName: finding.ownerSourceName,
151
+ resolved,
152
+ shouldRemove: (target) => target === finding.targetSourceName,
153
+ author: input.author,
154
+ trace: input.trace,
155
+ })));
156
+ }
157
+ const wikiFindings = input.findings.filter((finding) => finding.kind === 'missing_wiki_ref' ||
158
+ finding.kind === 'missing_wiki_sl_ref' ||
159
+ finding.kind === 'missing_wiki_body_sl_source' ||
160
+ finding.kind === 'missing_wiki_body_sl_entity' ||
161
+ finding.kind === 'missing_wiki_body_table');
162
+ const pageKeys = [...new Set(wikiFindings.map((finding) => finding.pageKey))].sort();
163
+ for (const pageKey of pageKeys) {
164
+ const page = input.wikiService ? await input.wikiService.readPage('GLOBAL', null, pageKey) : null;
165
+ if (!page) {
166
+ continue;
167
+ }
168
+ const frontmatter = { ...page.frontmatter };
169
+ let content = page.content;
170
+ let changed = false;
171
+ for (const finding of wikiFindings.filter((candidate) => candidate.pageKey === pageKey)) {
172
+ if (finding.kind === 'missing_wiki_ref') {
173
+ const refs = Array.isArray(frontmatter.refs) ? frontmatter.refs.filter((ref) => ref !== finding.targetPageKey) : [];
174
+ const nextContent = removeWikiRefToken(content, finding.targetPageKey);
175
+ if ((Array.isArray(frontmatter.refs) && refs.length !== frontmatter.refs.length) || nextContent !== content) {
176
+ if (Array.isArray(frontmatter.refs)) {
177
+ frontmatter.refs = refs;
178
+ }
179
+ content = nextContent;
180
+ changed = true;
181
+ const record = {
182
+ kind: 'wiki_ref',
183
+ artifact: `wiki/global/${pageKey}`,
184
+ removedRef: finding.targetPageKey,
185
+ absentTarget: finding.targetPageKey,
186
+ };
187
+ prunedReferences.push(record);
188
+ await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
189
+ }
190
+ }
191
+ else if (finding.kind === 'missing_wiki_sl_ref') {
192
+ const slRefs = Array.isArray(frontmatter.sl_refs)
193
+ ? frontmatter.sl_refs.filter((ref) => ref !== finding.ref)
194
+ : [];
195
+ if (Array.isArray(frontmatter.sl_refs) && slRefs.length !== frontmatter.sl_refs.length) {
196
+ frontmatter.sl_refs = slRefs;
197
+ changed = true;
198
+ const record = {
199
+ kind: 'wiki_sl_ref',
200
+ artifact: `wiki/global/${pageKey}`,
201
+ removedRef: finding.ref,
202
+ absentTarget: finding.sourceName,
203
+ };
204
+ prunedReferences.push(record);
205
+ await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
206
+ }
207
+ }
208
+ else {
209
+ const nextContent = removeInlineToken(content, finding.rawToken);
210
+ if (nextContent !== content) {
211
+ content = nextContent;
212
+ changed = true;
213
+ const record = {
214
+ kind: 'wiki_body_ref',
215
+ artifact: `wiki/global/${pageKey}`,
216
+ removedRef: finding.rawToken,
217
+ absentTarget: wikiBodyAbsentTarget(finding),
218
+ };
219
+ prunedReferences.push(record);
220
+ await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
221
+ }
222
+ }
223
+ }
224
+ if (changed && input.wikiService) {
225
+ await input.wikiService.writePage('GLOBAL', null, pageKey, frontmatter, content, input.author.name, input.author.email, `Prune dangling refs from ${pageKey}`, { skipLock: true });
226
+ }
227
+ }
228
+ return { prunedReferences, droppedSources };
229
+ }
@@ -8,6 +8,8 @@ export declare class IngestBundleRunner {
8
8
  private readonly logger;
9
9
  private readonly chainByConnection;
10
10
  constructor(deps: IngestBundleRunnerDeps);
11
+ private cachedWorkUnitOutcome;
12
+ private saveSuccessfulWorkUnitCache;
11
13
  run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult>;
12
14
  private formatRateLimitWait;
13
15
  private subscribeRateLimitGovernor;
@@ -61,6 +63,7 @@ export declare class IngestBundleRunner {
61
63
  private uniqueWikiPageKeys;
62
64
  private uniqueTouchedSlSources;
63
65
  private removedWikiPageKeysFromActions;
66
+ private markFinalGateDroppedSourceWorkUnits;
64
67
  private finalGateActionOrigins;
65
68
  private wikiPageKeysForFinalGates;
66
69
  private runWorkUnitInWorktree;