@kaelio/ktx 0.13.1 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/assets/python/{kaelio_ktx-0.13.1-py3-none-any.whl → kaelio_ktx-0.14.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-program.js +1 -1
  5. package/dist/commands/ingest-commands.d.ts +9 -0
  6. package/dist/commands/ingest-commands.js +37 -1
  7. package/dist/commands/knowledge-commands.js +3 -0
  8. package/dist/commands/setup-commands.js +14 -1
  9. package/dist/connection-drivers.d.ts +2 -0
  10. package/dist/connection-drivers.js +7 -3
  11. package/dist/connection.d.ts +3 -0
  12. package/dist/connection.js +27 -0
  13. package/dist/connectors/bigquery/connector.d.ts +17 -6
  14. package/dist/connectors/bigquery/connector.js +152 -76
  15. package/dist/connectors/bigquery/dialect.d.ts +2 -2
  16. package/dist/connectors/clickhouse/connector.d.ts +1 -0
  17. package/dist/connectors/clickhouse/connector.js +45 -16
  18. package/dist/connectors/clickhouse/dialect.d.ts +2 -2
  19. package/dist/connectors/mongodb/connector.d.ts +69 -0
  20. package/dist/connectors/mongodb/connector.js +307 -0
  21. package/dist/connectors/mongodb/dialect.d.ts +17 -0
  22. package/dist/connectors/mongodb/dialect.js +50 -0
  23. package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
  24. package/dist/connectors/mongodb/live-database-introspection.js +24 -0
  25. package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
  26. package/dist/connectors/mongodb/schema-inference.js +110 -0
  27. package/dist/connectors/mysql/connector.d.ts +1 -0
  28. package/dist/connectors/mysql/connector.js +18 -2
  29. package/dist/connectors/mysql/dialect.d.ts +2 -2
  30. package/dist/connectors/postgres/connector.d.ts +1 -0
  31. package/dist/connectors/postgres/connector.js +20 -3
  32. package/dist/connectors/postgres/dialect.d.ts +2 -2
  33. package/dist/connectors/snowflake/connector.d.ts +1 -0
  34. package/dist/connectors/snowflake/connector.js +27 -3
  35. package/dist/connectors/snowflake/dialect.d.ts +2 -2
  36. package/dist/connectors/sqlite/connector.d.ts +11 -1
  37. package/dist/connectors/sqlite/connector.js +120 -17
  38. package/dist/connectors/sqlite/dialect.d.ts +2 -2
  39. package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
  40. package/dist/connectors/sqlite/read-query-child.js +23 -0
  41. package/dist/connectors/sqlserver/connector.d.ts +2 -0
  42. package/dist/connectors/sqlserver/connector.js +20 -3
  43. package/dist/connectors/sqlserver/dialect.d.ts +2 -2
  44. package/dist/context/cache/content-result-cache.d.ts +36 -0
  45. package/dist/context/cache/content-result-cache.js +14 -0
  46. package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
  47. package/dist/context/cache/sqlite-content-result-cache.js +223 -0
  48. package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
  49. package/dist/context/connections/bigquery-identifiers.js +7 -0
  50. package/dist/context/connections/configured-connections.d.ts +9 -0
  51. package/dist/context/connections/configured-connections.js +18 -0
  52. package/dist/context/connections/dialects.d.ts +30 -4
  53. package/dist/context/connections/dialects.js +38 -11
  54. package/dist/context/connections/drivers.js +21 -0
  55. package/dist/context/connections/gdrive-config.d.ts +19 -0
  56. package/dist/context/connections/gdrive-config.js +57 -0
  57. package/dist/context/connections/query-deadline.d.ts +31 -0
  58. package/dist/context/connections/query-deadline.js +37 -0
  59. package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
  60. package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
  61. package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
  62. package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
  63. package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
  64. package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
  65. package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
  66. package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
  67. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
  68. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
  69. package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
  70. package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
  71. package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
  72. package/dist/context/ingest/adapters/gdrive/types.js +36 -0
  73. package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
  74. package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
  75. package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
  76. package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
  77. package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
  78. package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
  79. package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
  80. package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
  81. package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
  82. package/dist/context/ingest/adapters/live-database/stage.js +5 -5
  83. package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
  84. package/dist/context/ingest/adapters/metabase/types.js +1 -1
  85. package/dist/context/ingest/artifact-gates.d.ts +32 -1
  86. package/dist/context/ingest/artifact-gates.js +85 -18
  87. package/dist/context/ingest/final-gate-prune.d.ts +35 -0
  88. package/dist/context/ingest/final-gate-prune.js +229 -0
  89. package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
  90. package/dist/context/ingest/ingest-bundle.runner.js +459 -240
  91. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
  92. package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
  93. package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
  94. package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
  95. package/dist/context/ingest/local-adapters.js +6 -0
  96. package/dist/context/ingest/local-bundle-runtime.js +7 -2
  97. package/dist/context/ingest/local-ingest.js +1 -1
  98. package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
  99. package/dist/context/ingest/local-stage-ingest.js +3 -1
  100. package/dist/context/ingest/ports.d.ts +3 -0
  101. package/dist/context/ingest/report-snapshot.js +13 -3
  102. package/dist/context/ingest/reports.d.ts +3 -3
  103. package/dist/context/ingest/reports.js +3 -1
  104. package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
  105. package/dist/context/ingest/stages/build-wu-context.js +2 -1
  106. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
  107. package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
  108. package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
  109. package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
  110. package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
  111. package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
  112. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
  113. package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
  114. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
  115. package/dist/context/ingest/types.d.ts +3 -0
  116. package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
  117. package/dist/context/ingest/wiki-body-refs.js +37 -8
  118. package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
  119. package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
  120. package/dist/context/ingest/work-unit-cache.d.ts +67 -0
  121. package/dist/context/ingest/work-unit-cache.js +230 -0
  122. package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
  123. package/dist/context/llm/ai-sdk-runtime.js +5 -0
  124. package/dist/context/llm/claude-code-runtime.d.ts +4 -1
  125. package/dist/context/llm/claude-code-runtime.js +38 -8
  126. package/dist/context/llm/codex-runtime.d.ts +4 -1
  127. package/dist/context/llm/codex-runtime.js +48 -35
  128. package/dist/context/llm/runtime-port.d.ts +26 -0
  129. package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
  130. package/dist/context/llm/subprocess-generate-object-child.js +34 -0
  131. package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
  132. package/dist/context/llm/subprocess-generate-object.js +122 -0
  133. package/dist/context/mcp/context-tools.d.ts +2 -0
  134. package/dist/context/mcp/context-tools.js +111 -17
  135. package/dist/context/mcp/local-project-ports.d.ts +7 -0
  136. package/dist/context/mcp/local-project-ports.js +29 -0
  137. package/dist/context/mcp/logger.d.ts +24 -0
  138. package/dist/context/mcp/logger.js +49 -0
  139. package/dist/context/mcp/server.js +2 -0
  140. package/dist/context/mcp/types.d.ts +17 -0
  141. package/dist/context/memory/memory-agent.service.js +1 -1
  142. package/dist/context/project/config.d.ts +42 -0
  143. package/dist/context/project/config.js +5 -0
  144. package/dist/context/project/driver-schemas.d.ts +20 -0
  145. package/dist/context/project/driver-schemas.js +50 -1
  146. package/dist/context/project/project.js +1 -1
  147. package/dist/context/project/setup-config.js +1 -0
  148. package/dist/context/scan/description-generation.d.ts +4 -0
  149. package/dist/context/scan/description-generation.js +100 -13
  150. package/dist/context/scan/enabled-tables.d.ts +5 -0
  151. package/dist/context/scan/enabled-tables.js +13 -1
  152. package/dist/context/scan/enrichment-state.d.ts +50 -7
  153. package/dist/context/scan/enrichment-state.js +30 -13
  154. package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
  155. package/dist/context/scan/local-enrichment-artifacts.js +155 -32
  156. package/dist/context/scan/local-enrichment.d.ts +39 -4
  157. package/dist/context/scan/local-enrichment.js +345 -92
  158. package/dist/context/scan/local-scan.d.ts +4 -1
  159. package/dist/context/scan/local-scan.js +54 -13
  160. package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
  161. package/dist/context/scan/local-structural-artifacts.js +5 -0
  162. package/dist/context/scan/object-introspection.d.ts +21 -0
  163. package/dist/context/scan/object-introspection.js +35 -0
  164. package/dist/context/scan/relationship-benchmarks.js +5 -3
  165. package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
  166. package/dist/context/scan/relationship-composite-candidates.js +13 -1
  167. package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
  168. package/dist/context/scan/relationship-detection-budget.js +53 -0
  169. package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
  170. package/dist/context/scan/relationship-diagnostics.js +2 -0
  171. package/dist/context/scan/relationship-discovery.d.ts +9 -3
  172. package/dist/context/scan/relationship-discovery.js +27 -4
  173. package/dist/context/scan/relationship-llm-proposal.js +36 -27
  174. package/dist/context/scan/relationship-profiling.d.ts +7 -3
  175. package/dist/context/scan/relationship-profiling.js +53 -41
  176. package/dist/context/scan/relationship-validation.d.ts +6 -4
  177. package/dist/context/scan/relationship-validation.js +46 -32
  178. package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
  179. package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
  180. package/dist/context/scan/types.d.ts +5 -3
  181. package/dist/context/scan/types.js +2 -0
  182. package/dist/context/sl/local-query.js +5 -0
  183. package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
  184. package/dist/context/sl/semantic-layer.service.js +1 -1
  185. package/dist/context/sl/source-files.d.ts +5 -0
  186. package/dist/context/sl/source-files.js +17 -11
  187. package/dist/context/sl/tools/connection-id-schema.js +1 -1
  188. package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
  189. package/dist/context/sql-analysis/dialect-notes.js +40 -0
  190. package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
  191. package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
  192. package/dist/context/sql-analysis/dialects/mysql.md +9 -0
  193. package/dist/context/sql-analysis/dialects/postgres.md +10 -0
  194. package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
  195. package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
  196. package/dist/context/sql-analysis/dialects/tsql.md +10 -0
  197. package/dist/context/wiki/keys.js +1 -1
  198. package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
  199. package/dist/context/wiki/knowledge-wiki.service.js +2 -1
  200. package/dist/context/wiki/local-knowledge.d.ts +13 -0
  201. package/dist/context/wiki/local-knowledge.js +50 -6
  202. package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
  203. package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
  204. package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
  205. package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
  206. package/dist/context/wiki/types.d.ts +6 -0
  207. package/dist/context-build-view.d.ts +3 -0
  208. package/dist/context-build-view.js +1 -0
  209. package/dist/knowledge.d.ts +2 -0
  210. package/dist/knowledge.js +12 -1
  211. package/dist/local-adapters.js +11 -0
  212. package/dist/mcp-http-server.js +15 -1
  213. package/dist/mcp-server-factory.d.ts +2 -0
  214. package/dist/mcp-server-factory.js +15 -1
  215. package/dist/mcp-stdio-server.js +10 -2
  216. package/dist/notion-page-picker.js +1 -1
  217. package/dist/prompts/memory_agent_external_ingest.md +2 -0
  218. package/dist/public-ingest.d.ts +3 -1
  219. package/dist/public-ingest.js +3 -0
  220. package/dist/scan.d.ts +3 -1
  221. package/dist/scan.js +7 -0
  222. package/dist/setup-databases.d.ts +1 -1
  223. package/dist/setup-databases.js +43 -1
  224. package/dist/setup-sources.d.ts +5 -1
  225. package/dist/setup-sources.js +124 -48
  226. package/dist/setup.d.ts +3 -0
  227. package/dist/setup.js +6 -1
  228. package/dist/skills/analytics/SKILL.md +200 -2
  229. package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
  230. package/dist/skills/wiki_capture/SKILL.md +24 -0
  231. package/dist/sql.js +4 -0
  232. package/dist/status-project.d.ts +4 -0
  233. package/dist/status-project.js +54 -2
  234. package/dist/telemetry/events.d.ts +2 -2
  235. package/dist/text-ingest.d.ts +4 -0
  236. package/dist/text-ingest.js +58 -29
  237. package/dist/verbatim-ingest.d.ts +46 -0
  238. package/dist/verbatim-ingest.js +193 -0
  239. package/package.json +5 -1
  240. package/dist/context/ingest/final-gate-repair.d.ts +0 -28
  241. package/dist/context/ingest/final-gate-repair.js +0 -91
@@ -0,0 +1,153 @@
1
+ import { z } from 'zod';
2
+ export declare const GDRIVE_SCOPES: readonly ["https://www.googleapis.com/auth/drive.readonly", "https://www.googleapis.com/auth/documents.readonly"];
3
+ export declare const GDRIVE_SOURCE_KEY = "gdrive";
4
+ export declare const GDRIVE_DOC_MIME_TYPE = "application/vnd.google-apps.document";
5
+ export declare const GDRIVE_FOLDER_MIME_TYPE = "application/vnd.google-apps.folder";
6
+ export declare const gdrivePullConfigSchema: z.ZodObject<{
7
+ serviceAccountKey: z.ZodString;
8
+ folderId: z.ZodString;
9
+ recursive: z.ZodDefault<z.ZodBoolean>;
10
+ }, z.core.$strip>;
11
+ export type GdrivePullConfig = z.infer<typeof gdrivePullConfigSchema>;
12
+ export declare const gdriveManifestSchema: z.ZodObject<{
13
+ source: z.ZodLiteral<"gdrive">;
14
+ folderId: z.ZodString;
15
+ recursive: z.ZodBoolean;
16
+ fetchedAt: z.ZodString;
17
+ fileCount: z.ZodNumber;
18
+ skipped: z.ZodDefault<z.ZodArray<z.ZodObject<{
19
+ externalId: z.ZodString;
20
+ reason: z.ZodString;
21
+ }, z.core.$strip>>>;
22
+ warnings: z.ZodDefault<z.ZodArray<z.ZodString>>;
23
+ }, z.core.$strip>;
24
+ export type GdriveManifest = z.infer<typeof gdriveManifestSchema>;
25
+ export declare const gdriveMetadataSchema: z.ZodObject<{
26
+ id: z.ZodString;
27
+ title: z.ZodString;
28
+ path: z.ZodString;
29
+ url: z.ZodDefault<z.ZodNullable<z.ZodString>>;
30
+ mimeType: z.ZodLiteral<"application/vnd.google-apps.document">;
31
+ folderId: z.ZodString;
32
+ drivePath: z.ZodDefault<z.ZodArray<z.ZodString>>;
33
+ modifiedTime: z.ZodDefault<z.ZodNullable<z.ZodString>>;
34
+ }, z.core.$strip>;
35
+ export declare const gdriveServiceAccountKeySchema: z.ZodObject<{
36
+ client_email: z.ZodString;
37
+ private_key: z.ZodString;
38
+ project_id: z.ZodOptional<z.ZodString>;
39
+ }, z.core.$strip>;
40
+ export type GdriveServiceAccountKey = z.infer<typeof gdriveServiceAccountKeySchema>;
41
+ export interface GdriveFileRecord {
42
+ id: string;
43
+ name: string;
44
+ mimeType: string;
45
+ parents: string[];
46
+ webViewLink: string | null;
47
+ modifiedTime: string | null;
48
+ }
49
+ export interface GoogleDocsDocument {
50
+ documentId?: string;
51
+ title?: string;
52
+ body?: {
53
+ content?: GoogleDocsStructuralElement[];
54
+ };
55
+ documentStyle?: GoogleDocsDocumentStyle;
56
+ lists?: Record<string, GoogleDocsList>;
57
+ headers?: Record<string, GoogleDocsHeaderFooter>;
58
+ footers?: Record<string, GoogleDocsHeaderFooter>;
59
+ tabs?: GoogleDocsTab[];
60
+ }
61
+ export interface GoogleDocsList {
62
+ listProperties?: {
63
+ nestingLevels?: GoogleDocsListNestingLevel[];
64
+ };
65
+ }
66
+ interface GoogleDocsListNestingLevel {
67
+ glyphType?: string;
68
+ glyphSymbol?: string;
69
+ }
70
+ export interface GoogleDocsTab {
71
+ tabProperties?: {
72
+ tabId?: string;
73
+ title?: string;
74
+ };
75
+ childTabs?: GoogleDocsTab[];
76
+ documentTab?: {
77
+ body?: {
78
+ content?: GoogleDocsStructuralElement[];
79
+ };
80
+ documentStyle?: GoogleDocsDocumentStyle;
81
+ lists?: Record<string, GoogleDocsList>;
82
+ headers?: Record<string, GoogleDocsHeaderFooter>;
83
+ footers?: Record<string, GoogleDocsHeaderFooter>;
84
+ };
85
+ }
86
+ export interface GoogleDocsDocumentStyle {
87
+ defaultHeaderId?: string;
88
+ defaultFooterId?: string;
89
+ firstPageHeaderId?: string;
90
+ firstPageFooterId?: string;
91
+ evenPageHeaderId?: string;
92
+ evenPageFooterId?: string;
93
+ }
94
+ export interface GoogleDocsHeaderFooter {
95
+ headerId?: string;
96
+ footerId?: string;
97
+ content?: GoogleDocsStructuralElement[];
98
+ }
99
+ export interface GoogleDocsStructuralElement {
100
+ paragraph?: GoogleDocsParagraph;
101
+ table?: GoogleDocsTable;
102
+ sectionBreak?: unknown;
103
+ }
104
+ export interface GoogleDocsTable {
105
+ tableRows?: GoogleDocsTableRow[];
106
+ }
107
+ interface GoogleDocsTableRow {
108
+ tableCells?: GoogleDocsTableCell[];
109
+ }
110
+ export interface GoogleDocsTableCell {
111
+ content?: GoogleDocsStructuralElement[];
112
+ }
113
+ export interface GoogleDocsParagraph {
114
+ elements?: GoogleDocsParagraphElement[];
115
+ bullet?: {
116
+ listId?: string;
117
+ nestingLevel?: number;
118
+ };
119
+ paragraphStyle?: {
120
+ namedStyleType?: string;
121
+ headingId?: string;
122
+ };
123
+ }
124
+ export interface GoogleDocsLinkTarget {
125
+ id?: string;
126
+ tabId?: string;
127
+ }
128
+ export interface GoogleDocsParagraphElement {
129
+ textRun?: {
130
+ content?: string;
131
+ textStyle?: {
132
+ bold?: boolean;
133
+ italic?: boolean;
134
+ underline?: boolean;
135
+ strikethrough?: boolean;
136
+ link?: {
137
+ url?: string;
138
+ tabId?: string;
139
+ headingId?: string;
140
+ bookmarkId?: string;
141
+ heading?: GoogleDocsLinkTarget;
142
+ bookmark?: GoogleDocsLinkTarget;
143
+ };
144
+ weightedFontFamily?: {
145
+ fontFamily?: string;
146
+ };
147
+ baselineOffset?: 'SUPERSCRIPT' | 'SUBSCRIPT' | string;
148
+ };
149
+ };
150
+ inlineObjectElement?: unknown;
151
+ pageBreak?: unknown;
152
+ }
153
+ export {};
@@ -0,0 +1,36 @@
1
+ import { z } from 'zod';
2
+ const GDRIVE_DOCS_SCOPE = 'https://www.googleapis.com/auth/documents.readonly';
3
+ const GDRIVE_DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.readonly';
4
+ export const GDRIVE_SCOPES = [GDRIVE_DRIVE_SCOPE, GDRIVE_DOCS_SCOPE];
5
+ export const GDRIVE_SOURCE_KEY = 'gdrive';
6
+ export const GDRIVE_DOC_MIME_TYPE = 'application/vnd.google-apps.document';
7
+ export const GDRIVE_FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder';
8
+ export const gdrivePullConfigSchema = z.object({
9
+ serviceAccountKey: z.string().min(1),
10
+ folderId: z.string().min(1),
11
+ recursive: z.boolean().default(false),
12
+ });
13
+ export const gdriveManifestSchema = z.object({
14
+ source: z.literal(GDRIVE_SOURCE_KEY),
15
+ folderId: z.string().min(1),
16
+ recursive: z.boolean(),
17
+ fetchedAt: z.string().datetime(),
18
+ fileCount: z.number().int().nonnegative(),
19
+ skipped: z.array(z.object({ externalId: z.string(), reason: z.string() })).default([]),
20
+ warnings: z.array(z.string()).default([]),
21
+ });
22
+ export const gdriveMetadataSchema = z.object({
23
+ id: z.string(),
24
+ title: z.string(),
25
+ path: z.string(),
26
+ url: z.string().nullable().default(null),
27
+ mimeType: z.literal(GDRIVE_DOC_MIME_TYPE),
28
+ folderId: z.string(),
29
+ drivePath: z.array(z.string()).default([]),
30
+ modifiedTime: z.string().datetime().nullable().default(null),
31
+ });
32
+ export const gdriveServiceAccountKeySchema = z.object({
33
+ client_email: z.string().email(),
34
+ private_key: z.string().min(1),
35
+ project_id: z.string().min(1).optional(),
36
+ });
@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process';
2
2
  import { request as httpRequest } from 'node:http';
3
3
  import { request as httpsRequest } from 'node:https';
4
4
  import { URL } from 'node:url';
5
+ import { isKtxScanWarningCode } from '../../../scan/local-structural-artifacts.js';
5
6
  import { tableRefFromKey } from '../../../scan/table-ref.js';
6
7
  import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
7
8
  const DEFAULT_SCHEMAS = ['public'];
@@ -147,7 +148,29 @@ function mapTable(raw) {
147
148
  foreignKeys: recordArray(raw.foreign_keys).map(mapForeignKey),
148
149
  };
149
150
  }
151
+ function mapWarning(raw) {
152
+ const code = optionalString(raw.code);
153
+ // Drop codes Node cannot render, keeping the daemon and Node warning catalogs
154
+ // in parity rather than surfacing an unknown code downstream.
155
+ if (!code || !isKtxScanWarningCode(code))
156
+ return null;
157
+ const table = optionalString(raw.table);
158
+ const column = optionalString(raw.column);
159
+ return {
160
+ code,
161
+ message: requiredString(raw.message, 'warnings[].message'),
162
+ recoverable: raw.recoverable !== false,
163
+ ...(table ? { table } : {}),
164
+ ...(column ? { column } : {}),
165
+ ...(raw.metadata && typeof raw.metadata === 'object' && !Array.isArray(raw.metadata)
166
+ ? { metadata: recordValue(raw.metadata) }
167
+ : {}),
168
+ };
169
+ }
150
170
  function mapDaemonSnapshot(raw, input) {
171
+ const warnings = recordArray(raw.warnings)
172
+ .map(mapWarning)
173
+ .filter((warning) => warning !== null);
151
174
  return {
152
175
  connectionId: requiredString(raw.connection_id, 'connection_id') || input.connectionId,
153
176
  driver: 'postgres',
@@ -155,6 +178,7 @@ function mapDaemonSnapshot(raw, input) {
155
178
  scope: { schemas: input.schemas },
156
179
  metadata: recordValue(raw.metadata),
157
180
  tables: recordArray(raw.tables).map(mapTable),
181
+ ...(warnings.length > 0 ? { warnings } : {}),
158
182
  };
159
183
  }
160
184
  function serializeTableScope(options) {
@@ -0,0 +1,8 @@
1
+ import type { SourceFetchReport } from '../../types.js';
2
+ /**
3
+ * Derives the fetch report from the staged `warnings.json`: objects that failed
4
+ * introspection become `skipped` entries so the run report, ingest summary, and
5
+ * `ktx status` can surface them. Returns null when nothing was skipped, keeping
6
+ * clean ingests free of an empty report.
7
+ */
8
+ export declare function readLiveDatabaseFetchReport(stagedDir: string): Promise<SourceFetchReport | null>;
@@ -0,0 +1,37 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+ import { LIVE_DATABASE_WARNINGS_FILE } from './stage.js';
4
+ const OBJECT_SKIP_CODE = 'object_introspection_failed';
5
+ /**
6
+ * Derives the fetch report from the staged `warnings.json`: objects that failed
7
+ * introspection become `skipped` entries so the run report, ingest summary, and
8
+ * `ktx status` can surface them. Returns null when nothing was skipped, keeping
9
+ * clean ingests free of an empty report.
10
+ */
11
+ export async function readLiveDatabaseFetchReport(stagedDir) {
12
+ let parsed;
13
+ try {
14
+ parsed = JSON.parse(await readFile(join(stagedDir, LIVE_DATABASE_WARNINGS_FILE), 'utf8'));
15
+ }
16
+ catch {
17
+ return null;
18
+ }
19
+ const warnings = parsed && typeof parsed === 'object' && Array.isArray(parsed.warnings)
20
+ ? (parsed.warnings)
21
+ : [];
22
+ const skipped = warnings
23
+ .filter((warning) => warning.code === OBJECT_SKIP_CODE)
24
+ .map((warning) => ({
25
+ rawPath: '',
26
+ entityType: 'database_object',
27
+ entityId: typeof warning.table === 'string' ? warning.table : null,
28
+ severity: 'warning',
29
+ statusCode: null,
30
+ message: typeof warning.message === 'string' ? warning.message : 'introspection failed',
31
+ retryRecommended: false,
32
+ }));
33
+ if (skipped.length === 0) {
34
+ return null;
35
+ }
36
+ return { status: 'partial', retryRecommended: false, skipped, warnings: [] };
37
+ }
@@ -1,4 +1,4 @@
1
- import type { ChunkResult, DiffSet, FetchContext, SourceAdapter } from '../../types.js';
1
+ import type { ChunkResult, DiffSet, FetchContext, SourceAdapter, SourceFetchReport } from '../../types.js';
2
2
  import type { LiveDatabaseSourceAdapterDeps } from './types.js';
3
3
  export declare class LiveDatabaseSourceAdapter implements SourceAdapter {
4
4
  private readonly deps;
@@ -6,6 +6,7 @@ export declare class LiveDatabaseSourceAdapter implements SourceAdapter {
6
6
  readonly skillNames: string[];
7
7
  constructor(deps: LiveDatabaseSourceAdapterDeps);
8
8
  detect(stagedDir: string): Promise<boolean>;
9
+ readFetchReport(stagedDir: string): Promise<SourceFetchReport | null>;
9
10
  fetch(_pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise<void>;
10
11
  chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
11
12
  }
@@ -1,4 +1,6 @@
1
1
  import { chunkLiveDatabaseStagedDir } from './chunk.js';
2
+ import { readLiveDatabaseFetchReport } from './fetch-report.js';
3
+ import { assertLiveDatabaseScanOutcome } from './scan-outcome.js';
2
4
  import { detectLiveDatabaseStagedDir, writeLiveDatabaseSnapshot } from './stage.js';
3
5
  export class LiveDatabaseSourceAdapter {
4
6
  deps;
@@ -10,14 +12,19 @@ export class LiveDatabaseSourceAdapter {
10
12
  detect(stagedDir) {
11
13
  return detectLiveDatabaseStagedDir(stagedDir);
12
14
  }
15
+ readFetchReport(stagedDir) {
16
+ return readLiveDatabaseFetchReport(stagedDir);
17
+ }
13
18
  async fetch(_pullConfig, stagedDir, ctx) {
14
19
  const tableScope = ctx.tableScope;
15
20
  const snapshot = await this.deps.introspection.extractSchema(ctx.connectionId, { tableScope });
16
- await writeLiveDatabaseSnapshot(stagedDir, {
21
+ const finalized = {
17
22
  ...snapshot,
18
23
  connectionId: ctx.connectionId,
19
24
  extractedAt: snapshot.extractedAt ?? (this.deps.now ?? (() => new Date()))().toISOString(),
20
- });
25
+ };
26
+ assertLiveDatabaseScanOutcome({ connectionId: ctx.connectionId, scope: tableScope, snapshot: finalized });
27
+ await writeLiveDatabaseSnapshot(stagedDir, finalized);
21
28
  }
22
29
  chunk(stagedDir, diffSet) {
23
30
  return chunkLiveDatabaseStagedDir(stagedDir, diffSet);
@@ -64,6 +64,8 @@ export interface BuildLiveDatabaseManifestShardsResult {
64
64
  }
65
65
  export declare function mergeUsagePreservingExternal(existing: TableUsageOutput | undefined, incoming: TableUsageOutput | undefined): TableUsageOutput | undefined;
66
66
  /** @internal */
67
+ export declare function buildTableRef(name: string, catalog: string | null, db: string | null): string;
68
+ /** @internal */
67
69
  export declare function buildJoinsByTable(tableNames: Set<string>, joins: LiveDatabaseManifestJoinData[], preservedJoins: Map<string, LiveDatabaseManifestJoinEntry[]>, federatedSiblingTargets?: Set<string>): Map<string, LiveDatabaseManifestJoinEntry[]>;
68
70
  export declare function buildLiveDatabaseManifestShards(input: BuildLiveDatabaseManifestShardsInput): BuildLiveDatabaseManifestShardsResult;
69
71
  export {};
@@ -73,7 +73,8 @@ function getShardKey(connectionType, catalog, db) {
73
73
  }
74
74
  }
75
75
  }
76
- function buildTableRef(name, catalog, db) {
76
+ /** @internal */
77
+ export function buildTableRef(name, catalog, db) {
77
78
  const parts = [];
78
79
  if (catalog) {
79
80
  parts.push(catalog);
@@ -154,7 +155,10 @@ export function buildLiveDatabaseManifestShards(input) {
154
155
  for (const table of input.tables) {
155
156
  const shardKey = getShardKey(input.connectionType, table.catalog, table.db);
156
157
  const shard = shards.get(shardKey) ?? { tables: {} };
157
- const existingDescriptions = input.existingDescriptions?.get(table.name);
158
+ // Existing descriptions/usage are keyed by the fully-qualified ref so two
159
+ // same-named tables in different schemas never share an entry.
160
+ const fullRef = buildTableRef(table.name, table.catalog, table.db);
161
+ const existingDescriptions = input.existingDescriptions?.get(fullRef);
158
162
  const columns = table.columns.map((column) => {
159
163
  const manifestColumn = {
160
164
  name: column.name,
@@ -173,14 +177,14 @@ export function buildLiveDatabaseManifestShards(input) {
173
177
  return manifestColumn;
174
178
  });
175
179
  const entry = {
176
- table: buildTableRef(table.name, table.catalog, table.db),
180
+ table: fullRef,
177
181
  columns,
178
182
  };
179
183
  const tableDescriptions = mergeDescriptionsPreservingExternal(existingDescriptions?.table, table.descriptions);
180
184
  if (tableDescriptions) {
181
185
  entry.descriptions = tableDescriptions;
182
186
  }
183
- const usage = mergeUsagePreservingExternal(input.existingUsage?.get(table.name), table.usage);
187
+ const usage = mergeUsagePreservingExternal(input.existingUsage?.get(fullRef), table.usage);
184
188
  if (usage) {
185
189
  entry.usage = usage;
186
190
  }
@@ -0,0 +1,17 @@
1
+ import { type KtxTableRefKey } from '../../../scan/table-ref.js';
2
+ import type { KtxSchemaSnapshot } from '../../../scan/types.js';
3
+ /**
4
+ * Enforces the partial-vs-total outcome rules for a live-database snapshot,
5
+ * uniformly for every connector. Outcomes follow from object counts, not a
6
+ * mode: a connection with at least one ingested object succeeds (any broken
7
+ * objects ride along as warnings); a connection where every introspected object
8
+ * failed, or a non-empty enabled_tables scope that matched nothing, raises a
9
+ * clear connection error instead of staging an empty layer that would later
10
+ * surface as the generic "did not recognize" message. A legitimately empty
11
+ * database (no scope, no objects) succeeds with an empty layer.
12
+ */
13
+ export declare function assertLiveDatabaseScanOutcome(input: {
14
+ connectionId: string;
15
+ scope: ReadonlySet<KtxTableRefKey> | undefined;
16
+ snapshot: KtxSchemaSnapshot;
17
+ }): void;
@@ -0,0 +1,40 @@
1
+ import { KtxExpectedError } from '../../../../errors.js';
2
+ import { tableRefFromKey } from '../../../scan/table-ref.js';
3
+ const OBJECT_SKIP_CODE = 'object_introspection_failed';
4
+ function formatScopeEntry(key) {
5
+ const ref = tableRefFromKey(key);
6
+ return [ref.catalog, ref.db, ref.name].filter((part) => Boolean(part)).join('.');
7
+ }
8
+ function discoveredObjectNames(snapshot) {
9
+ const raw = snapshot.metadata.discovered_object_names;
10
+ return Array.isArray(raw) ? raw.filter((value) => typeof value === 'string') : [];
11
+ }
12
+ /**
13
+ * Enforces the partial-vs-total outcome rules for a live-database snapshot,
14
+ * uniformly for every connector. Outcomes follow from object counts, not a
15
+ * mode: a connection with at least one ingested object succeeds (any broken
16
+ * objects ride along as warnings); a connection where every introspected object
17
+ * failed, or a non-empty enabled_tables scope that matched nothing, raises a
18
+ * clear connection error instead of staging an empty layer that would later
19
+ * surface as the generic "did not recognize" message. A legitimately empty
20
+ * database (no scope, no objects) succeeds with an empty layer.
21
+ */
22
+ export function assertLiveDatabaseScanOutcome(input) {
23
+ const { connectionId, scope, snapshot } = input;
24
+ if (snapshot.tables.length > 0) {
25
+ return;
26
+ }
27
+ const skipped = (snapshot.warnings ?? []).filter((warning) => warning.code === OBJECT_SKIP_CODE);
28
+ if (skipped.length > 0) {
29
+ const detail = skipped.map((warning) => `${warning.table ?? 'object'}: ${warning.message}`).join('; ');
30
+ throw new KtxExpectedError(`Connection "${connectionId}" produced no semantic layer: all ${skipped.length} introspected ` +
31
+ `${skipped.length === 1 ? 'object' : 'objects'} failed (${detail}).`);
32
+ }
33
+ if (scope && scope.size > 0) {
34
+ const requested = [...scope].map(formatScopeEntry).sort();
35
+ const available = discoveredObjectNames(snapshot);
36
+ const availableClause = available.length > 0 ? ` Available objects: ${available.join(', ')}.` : '';
37
+ throw new KtxExpectedError(`enabled_tables for connection "${connectionId}" matched no objects ` +
38
+ `(looked for: ${requested.join(', ')}).${availableClause}`);
39
+ }
40
+ }
@@ -101,13 +101,13 @@ export async function readLiveDatabaseTableFiles(stagedDir) {
101
101
  return out;
102
102
  }
103
103
  export async function detectLiveDatabaseStagedDir(stagedDir) {
104
+ // A valid live-database staging is identified by its connection.json marker.
105
+ // An empty table set is a legitimate outcome (an empty database), so the
106
+ // presence of table files is not required — the total-vs-partial decision is
107
+ // made earlier by assertLiveDatabaseScanOutcome, before staging.
104
108
  try {
105
109
  const meta = JSON.parse(await readFile(join(stagedDir, LIVE_DATABASE_META_FILE), 'utf8'));
106
- if (!meta || typeof meta !== 'object' || Array.isArray(meta)) {
107
- return false;
108
- }
109
- const files = await readLiveDatabaseTableFiles(stagedDir);
110
- return files.length > 0;
110
+ return Boolean(meta) && typeof meta === 'object' && !Array.isArray(meta);
111
111
  }
112
112
  catch {
113
113
  return false;
@@ -118,8 +118,8 @@ export declare const stagedSyncConfigSchema: z.ZodObject<{
118
118
  }>;
119
119
  selections: z.ZodArray<z.ZodObject<{
120
120
  selectionType: z.ZodEnum<{
121
- collection: "collection";
122
121
  item: "item";
122
+ collection: "collection";
123
123
  }>;
124
124
  metabaseObjectId: z.ZodNumber;
125
125
  }, z.core.$strip>>;
@@ -1,6 +1,6 @@
1
1
  import { z } from 'zod';
2
2
  const metabaseSyncModeSchema = z.enum(['ALL', 'ONLY', 'EXCEPT']);
3
- const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/);
3
+ const metabaseLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9_][a-zA-Z0-9_-]*$/);
4
4
  /**
5
5
  * The lean config the adapter needs at `fetch()` time. Lives in the BullMQ payload's
6
6
  * `bundleRef.config` when the runner invokes the adapter. Never persisted beyond one
@@ -2,6 +2,7 @@ import type { SemanticLayerService } from '../../context/sl/semantic-layer.servi
2
2
  import type { TouchedSlSource } from '../../context/tools/touched-sl-sources.js';
3
3
  import type { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
4
4
  import type { WuValidationResult } from './stages/validate-wu-sources.js';
5
+ import { type WikiBodyRefIssue } from './wiki-body-refs.js';
5
6
  export interface FinalArtifactGateInput {
6
7
  connectionIds: string[];
7
8
  changedWikiPageKeys: string[];
@@ -18,5 +19,35 @@ export interface ProvenanceRawPathValidationInput {
18
19
  currentRawPaths: Set<string>;
19
20
  deletedRawPaths: Set<string>;
20
21
  }
21
- export declare function validateFinalIngestArtifacts(input: FinalArtifactGateInput): Promise<void>;
22
+ export type FinalArtifactGateFinding = {
23
+ kind: 'invalid_source';
24
+ connectionId: string;
25
+ sourceName: string;
26
+ errors: string[];
27
+ } | {
28
+ kind: 'missing_join_target';
29
+ ownerConnectionId: string;
30
+ ownerSourceName: string;
31
+ targetSourceName: string;
32
+ message: string;
33
+ } | {
34
+ kind: 'missing_wiki_ref';
35
+ pageKey: string;
36
+ targetPageKey: string;
37
+ message: string;
38
+ } | {
39
+ kind: 'missing_wiki_sl_ref';
40
+ pageKey: string;
41
+ ref: string;
42
+ sourceName: string;
43
+ entityName: string | null;
44
+ message: string;
45
+ } | WikiBodyRefIssue;
46
+ export interface FinalArtifactGateResult {
47
+ ok: boolean;
48
+ findings: FinalArtifactGateFinding[];
49
+ }
50
+ export declare function formatFinalArtifactGateFindings(findings: FinalArtifactGateFinding[]): string;
51
+ export declare function isFinalArtifactGateFindingPruneable(finding: FinalArtifactGateFinding): boolean;
52
+ export declare function validateFinalIngestArtifacts(input: FinalArtifactGateInput): Promise<FinalArtifactGateResult>;
22
53
  export declare function validateProvenanceRawPaths(input: ProvenanceRawPathValidationInput): void;