@kaelio/ktx 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/assets/python/{kaelio_ktx-0.13.0-py3-none-any.whl → kaelio_ktx-0.14.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-program.js +1 -1
  5. package/dist/commands/ingest-commands.d.ts +9 -0
  6. package/dist/commands/ingest-commands.js +37 -1
  7. package/dist/commands/knowledge-commands.js +3 -0
  8. package/dist/commands/setup-commands.js +14 -1
  9. package/dist/connection-drivers.d.ts +2 -0
  10. package/dist/connection-drivers.js +7 -3
  11. package/dist/connection.d.ts +3 -0
  12. package/dist/connection.js +27 -0
  13. package/dist/connectors/bigquery/connector.d.ts +17 -6
  14. package/dist/connectors/bigquery/connector.js +152 -76
  15. package/dist/connectors/bigquery/dialect.d.ts +2 -2
  16. package/dist/connectors/clickhouse/connector.d.ts +1 -0
  17. package/dist/connectors/clickhouse/connector.js +45 -16
  18. package/dist/connectors/clickhouse/dialect.d.ts +2 -2
  19. package/dist/connectors/mongodb/connector.d.ts +69 -0
  20. package/dist/connectors/mongodb/connector.js +307 -0
  21. package/dist/connectors/mongodb/dialect.d.ts +17 -0
  22. package/dist/connectors/mongodb/dialect.js +50 -0
  23. package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
  24. package/dist/connectors/mongodb/live-database-introspection.js +24 -0
  25. package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
  26. package/dist/connectors/mongodb/schema-inference.js +110 -0
  27. package/dist/connectors/mysql/connector.d.ts +1 -0
  28. package/dist/connectors/mysql/connector.js +18 -2
  29. package/dist/connectors/mysql/dialect.d.ts +2 -2
  30. package/dist/connectors/postgres/connector.d.ts +1 -0
  31. package/dist/connectors/postgres/connector.js +20 -3
  32. package/dist/connectors/postgres/dialect.d.ts +2 -2
  33. package/dist/connectors/snowflake/connector.d.ts +1 -0
  34. package/dist/connectors/snowflake/connector.js +27 -3
  35. package/dist/connectors/snowflake/dialect.d.ts +2 -2
  36. package/dist/connectors/sqlite/connector.d.ts +11 -1
  37. package/dist/connectors/sqlite/connector.js +120 -17
  38. package/dist/connectors/sqlite/dialect.d.ts +2 -2
  39. package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
  40. package/dist/connectors/sqlite/read-query-child.js +23 -0
  41. package/dist/connectors/sqlserver/connector.d.ts +2 -0
  42. package/dist/connectors/sqlserver/connector.js +23 -5
  43. package/dist/connectors/sqlserver/dialect.d.ts +2 -2
  44. package/dist/context/cache/content-result-cache.d.ts +36 -0
  45. package/dist/context/cache/content-result-cache.js +14 -0
  46. package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
  47. package/dist/context/cache/sqlite-content-result-cache.js +223 -0
  48. package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
  49. package/dist/context/connections/bigquery-identifiers.js +7 -0
  50. package/dist/context/connections/configured-connections.d.ts +9 -0
  51. package/dist/context/connections/configured-connections.js +18 -0
  52. package/dist/context/connections/dialects.d.ts +30 -4
  53. package/dist/context/connections/dialects.js +38 -11
  54. package/dist/context/connections/drivers.js +21 -0
  55. package/dist/context/connections/gdrive-config.d.ts +19 -0
  56. package/dist/context/connections/gdrive-config.js +57 -0
  57. package/dist/context/connections/query-deadline.d.ts +31 -0
  58. package/dist/context/connections/query-deadline.js +37 -0
  59. package/dist/context/connections/read-only-sql.d.ts +5 -0
  60. package/dist/context/connections/read-only-sql.js +139 -1
  61. package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
  62. package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
  63. package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
  64. package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
  65. package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
  66. package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
  67. package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
  68. package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
  69. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
  70. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
  71. package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
  72. package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
  73. package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
  74. package/dist/context/ingest/adapters/gdrive/types.js +36 -0
  75. package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
  76. package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
  77. package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
  78. package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
  79. package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
  80. package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
  81. package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
  82. package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
  83. package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
  84. package/dist/context/ingest/adapters/live-database/stage.js +5 -5
  85. package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
  86. package/dist/context/ingest/adapters/metabase/types.js +1 -1
  87. package/dist/context/ingest/artifact-gates.d.ts +32 -1
  88. package/dist/context/ingest/artifact-gates.js +85 -18
  89. package/dist/context/ingest/final-gate-prune.d.ts +35 -0
  90. package/dist/context/ingest/final-gate-prune.js +229 -0
  91. package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
  92. package/dist/context/ingest/ingest-bundle.runner.js +459 -240
  93. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
  94. package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
  95. package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
  96. package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
  97. package/dist/context/ingest/local-adapters.js +6 -0
  98. package/dist/context/ingest/local-bundle-runtime.js +7 -2
  99. package/dist/context/ingest/local-ingest.js +1 -1
  100. package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
  101. package/dist/context/ingest/local-stage-ingest.js +3 -1
  102. package/dist/context/ingest/ports.d.ts +3 -0
  103. package/dist/context/ingest/report-snapshot.js +13 -3
  104. package/dist/context/ingest/reports.d.ts +3 -3
  105. package/dist/context/ingest/reports.js +3 -1
  106. package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
  107. package/dist/context/ingest/stages/build-wu-context.js +2 -1
  108. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
  109. package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
  110. package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
  111. package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
  112. package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
  113. package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
  114. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
  115. package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
  116. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
  117. package/dist/context/ingest/types.d.ts +3 -0
  118. package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
  119. package/dist/context/ingest/wiki-body-refs.js +37 -8
  120. package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
  121. package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
  122. package/dist/context/ingest/work-unit-cache.d.ts +67 -0
  123. package/dist/context/ingest/work-unit-cache.js +230 -0
  124. package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
  125. package/dist/context/llm/ai-sdk-runtime.js +5 -0
  126. package/dist/context/llm/claude-code-runtime.d.ts +4 -1
  127. package/dist/context/llm/claude-code-runtime.js +38 -8
  128. package/dist/context/llm/codex-runtime.d.ts +4 -1
  129. package/dist/context/llm/codex-runtime.js +48 -35
  130. package/dist/context/llm/runtime-port.d.ts +26 -0
  131. package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
  132. package/dist/context/llm/subprocess-generate-object-child.js +34 -0
  133. package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
  134. package/dist/context/llm/subprocess-generate-object.js +122 -0
  135. package/dist/context/mcp/context-tools.d.ts +2 -0
  136. package/dist/context/mcp/context-tools.js +111 -17
  137. package/dist/context/mcp/local-project-ports.d.ts +7 -0
  138. package/dist/context/mcp/local-project-ports.js +29 -0
  139. package/dist/context/mcp/logger.d.ts +24 -0
  140. package/dist/context/mcp/logger.js +49 -0
  141. package/dist/context/mcp/server.js +2 -0
  142. package/dist/context/mcp/types.d.ts +17 -0
  143. package/dist/context/memory/memory-agent.service.js +1 -1
  144. package/dist/context/project/config.d.ts +42 -0
  145. package/dist/context/project/config.js +5 -0
  146. package/dist/context/project/driver-schemas.d.ts +20 -0
  147. package/dist/context/project/driver-schemas.js +50 -1
  148. package/dist/context/project/project.js +1 -1
  149. package/dist/context/project/setup-config.js +1 -0
  150. package/dist/context/scan/description-generation.d.ts +4 -0
  151. package/dist/context/scan/description-generation.js +100 -13
  152. package/dist/context/scan/enabled-tables.d.ts +5 -0
  153. package/dist/context/scan/enabled-tables.js +13 -1
  154. package/dist/context/scan/enrichment-state.d.ts +50 -7
  155. package/dist/context/scan/enrichment-state.js +30 -13
  156. package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
  157. package/dist/context/scan/local-enrichment-artifacts.js +155 -32
  158. package/dist/context/scan/local-enrichment.d.ts +39 -4
  159. package/dist/context/scan/local-enrichment.js +345 -92
  160. package/dist/context/scan/local-scan.d.ts +4 -1
  161. package/dist/context/scan/local-scan.js +54 -13
  162. package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
  163. package/dist/context/scan/local-structural-artifacts.js +5 -0
  164. package/dist/context/scan/object-introspection.d.ts +21 -0
  165. package/dist/context/scan/object-introspection.js +35 -0
  166. package/dist/context/scan/relationship-benchmarks.js +5 -3
  167. package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
  168. package/dist/context/scan/relationship-composite-candidates.js +13 -1
  169. package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
  170. package/dist/context/scan/relationship-detection-budget.js +53 -0
  171. package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
  172. package/dist/context/scan/relationship-diagnostics.js +2 -0
  173. package/dist/context/scan/relationship-discovery.d.ts +9 -3
  174. package/dist/context/scan/relationship-discovery.js +27 -4
  175. package/dist/context/scan/relationship-llm-proposal.js +36 -27
  176. package/dist/context/scan/relationship-profiling.d.ts +7 -3
  177. package/dist/context/scan/relationship-profiling.js +53 -41
  178. package/dist/context/scan/relationship-validation.d.ts +6 -4
  179. package/dist/context/scan/relationship-validation.js +46 -32
  180. package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
  181. package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
  182. package/dist/context/scan/types.d.ts +5 -3
  183. package/dist/context/scan/types.js +2 -0
  184. package/dist/context/sl/local-query.js +5 -0
  185. package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
  186. package/dist/context/sl/semantic-layer.service.js +1 -1
  187. package/dist/context/sl/source-files.d.ts +5 -0
  188. package/dist/context/sl/source-files.js +17 -11
  189. package/dist/context/sl/tools/connection-id-schema.js +1 -1
  190. package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
  191. package/dist/context/sql-analysis/dialect-notes.js +40 -0
  192. package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
  193. package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
  194. package/dist/context/sql-analysis/dialects/mysql.md +9 -0
  195. package/dist/context/sql-analysis/dialects/postgres.md +10 -0
  196. package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
  197. package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
  198. package/dist/context/sql-analysis/dialects/tsql.md +10 -0
  199. package/dist/context/wiki/keys.js +1 -1
  200. package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
  201. package/dist/context/wiki/knowledge-wiki.service.js +2 -1
  202. package/dist/context/wiki/local-knowledge.d.ts +13 -0
  203. package/dist/context/wiki/local-knowledge.js +50 -6
  204. package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
  205. package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
  206. package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
  207. package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
  208. package/dist/context/wiki/types.d.ts +6 -0
  209. package/dist/context-build-view.d.ts +3 -0
  210. package/dist/context-build-view.js +1 -0
  211. package/dist/knowledge.d.ts +2 -0
  212. package/dist/knowledge.js +12 -1
  213. package/dist/local-adapters.js +11 -0
  214. package/dist/mcp-http-server.js +15 -1
  215. package/dist/mcp-server-factory.d.ts +2 -0
  216. package/dist/mcp-server-factory.js +15 -1
  217. package/dist/mcp-stdio-server.js +10 -2
  218. package/dist/notion-page-picker.js +1 -1
  219. package/dist/prompts/memory_agent_external_ingest.md +2 -0
  220. package/dist/public-ingest.d.ts +3 -1
  221. package/dist/public-ingest.js +3 -0
  222. package/dist/scan.d.ts +3 -1
  223. package/dist/scan.js +7 -0
  224. package/dist/setup-databases.d.ts +1 -1
  225. package/dist/setup-databases.js +43 -1
  226. package/dist/setup-sources.d.ts +5 -1
  227. package/dist/setup-sources.js +124 -48
  228. package/dist/setup.d.ts +3 -0
  229. package/dist/setup.js +6 -1
  230. package/dist/skills/analytics/SKILL.md +200 -2
  231. package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
  232. package/dist/skills/wiki_capture/SKILL.md +24 -0
  233. package/dist/sql.js +4 -0
  234. package/dist/status-project.d.ts +4 -0
  235. package/dist/status-project.js +54 -2
  236. package/dist/telemetry/events.d.ts +2 -2
  237. package/dist/text-ingest.d.ts +4 -0
  238. package/dist/text-ingest.js +58 -29
  239. package/dist/verbatim-ingest.d.ts +46 -0
  240. package/dist/verbatim-ingest.js +193 -0
  241. package/package.json +5 -1
  242. package/dist/context/ingest/final-gate-repair.d.ts +0 -28
  243. package/dist/context/ingest/final-gate-repair.js +0 -91
@@ -0,0 +1,256 @@
1
+ function escapeMarkdownText(value) {
2
+ return value.replace(/([*_~`])/g, '\\$1');
3
+ }
4
+ function normalizeInternalLinkTarget(prefix, target) {
5
+ const id = typeof target === 'string' ? target : target?.id;
6
+ if (!id?.trim()) {
7
+ return null;
8
+ }
9
+ return `#${prefix}-${id.trim()}`;
10
+ }
11
+ function resolveLinkHref(element) {
12
+ const link = element.textRun?.textStyle?.link;
13
+ const href = link?.url?.trim();
14
+ if (href) {
15
+ return href;
16
+ }
17
+ return (normalizeInternalLinkTarget('heading', link?.heading) ??
18
+ normalizeInternalLinkTarget('heading', link?.headingId) ??
19
+ normalizeInternalLinkTarget('bookmark', link?.bookmark) ??
20
+ normalizeInternalLinkTarget('bookmark', link?.bookmarkId) ??
21
+ null);
22
+ }
23
+ function normalizeTextRun(element) {
24
+ const content = element.textRun?.content ?? '';
25
+ const style = element.textRun?.textStyle;
26
+ let text = escapeMarkdownText(content.replace(/\r/g, ''));
27
+ if (!text && element.inlineObjectElement) {
28
+ return '[Embedded object]';
29
+ }
30
+ if (!text && element.pageBreak) {
31
+ return '\n---\n';
32
+ }
33
+ if (!text) {
34
+ return '';
35
+ }
36
+ const href = resolveLinkHref(element);
37
+ const isCode = style?.weightedFontFamily?.fontFamily === 'Courier New';
38
+ if (isCode) {
39
+ text = `\`${text.replace(/`/g, '\\`')}\``;
40
+ }
41
+ if (style?.bold) {
42
+ text = `**${text}**`;
43
+ }
44
+ if (style?.italic) {
45
+ text = `*${text}*`;
46
+ }
47
+ if (style?.underline) {
48
+ text = `<u>${text}</u>`;
49
+ }
50
+ if (style?.strikethrough) {
51
+ text = `~~${text}~~`;
52
+ }
53
+ if (href) {
54
+ text = `[${text}](${href.replace(/\)/g, '\\)')})`;
55
+ }
56
+ if (style?.baselineOffset === 'SUPERSCRIPT') {
57
+ text = `<sup>${text}</sup>`;
58
+ }
59
+ else if (style?.baselineOffset === 'SUBSCRIPT') {
60
+ text = `<sub>${text}</sub>`;
61
+ }
62
+ return text;
63
+ }
64
+ function paragraphText(paragraph) {
65
+ return (paragraph?.elements ?? [])
66
+ .map((element) => normalizeTextRun(element))
67
+ .join('')
68
+ .replace(/\n/g, '')
69
+ .trim();
70
+ }
71
+ function headingPrefix(namedStyleType) {
72
+ if (namedStyleType === 'TITLE') {
73
+ return '#';
74
+ }
75
+ if (namedStyleType === 'SUBTITLE') {
76
+ return '##';
77
+ }
78
+ if (!namedStyleType?.startsWith('HEADING_')) {
79
+ return null;
80
+ }
81
+ const level = Number.parseInt(namedStyleType.slice('HEADING_'.length), 10);
82
+ if (Number.isNaN(level) || level < 1) {
83
+ return null;
84
+ }
85
+ return '#'.repeat(Math.min(level, 6));
86
+ }
87
+ function isOrderedListLevel(level) {
88
+ const glyphType = level?.glyphType?.toUpperCase();
89
+ if (glyphType) {
90
+ return (glyphType.includes('NUMBER') ||
91
+ glyphType.includes('DECIMAL') ||
92
+ glyphType.includes('ALPHA') ||
93
+ glyphType.includes('ROMAN') ||
94
+ glyphType.includes('LATIN'));
95
+ }
96
+ const glyphSymbol = level?.glyphSymbol?.trim();
97
+ return glyphSymbol === '%0.' || glyphSymbol === '%0)' || glyphSymbol === '1.' || glyphSymbol === '1)';
98
+ }
99
+ function listPrefix(paragraph, lists) {
100
+ if (!paragraph.bullet) {
101
+ return null;
102
+ }
103
+ const level = Math.max(paragraph.bullet.nestingLevel ?? 0, 0);
104
+ const indent = ' '.repeat(level);
105
+ const listDefinition = paragraph.bullet.listId ? lists?.[paragraph.bullet.listId] : undefined;
106
+ const listLevel = listDefinition?.listProperties?.nestingLevels?.[level];
107
+ return `${indent}${isOrderedListLevel(listLevel) ? '1. ' : '- '}`;
108
+ }
109
+ function paragraphToMarkdown(paragraph, lists) {
110
+ const text = paragraphText(paragraph);
111
+ if (!text) {
112
+ return null;
113
+ }
114
+ const prefix = paragraph ? listPrefix(paragraph, lists) : null;
115
+ if (prefix) {
116
+ return `${prefix}${text}`;
117
+ }
118
+ const heading = headingPrefix(paragraph?.paragraphStyle?.namedStyleType);
119
+ if (heading) {
120
+ const headingLine = `${heading} ${text}`;
121
+ const headingId = paragraph?.paragraphStyle?.headingId?.trim();
122
+ return headingId ? `<a id="heading-${headingId}"></a>\n${headingLine}` : headingLine;
123
+ }
124
+ return text;
125
+ }
126
+ function normalizeTableCell(cell, lists) {
127
+ const blocks = normalizeStructuralElements(cell?.content ?? [], lists);
128
+ return blocks
129
+ .map((block) => block.replace(/\n/g, ' <br> '))
130
+ .join(' / ')
131
+ .replace(/\|/g, '\\|')
132
+ .trim();
133
+ }
134
+ function markdownTableDivider(columnCount) {
135
+ return `| ${Array.from({ length: columnCount }, () => '---').join(' | ')} |`;
136
+ }
137
+ function normalizeTable(table, lists) {
138
+ const rows = table?.tableRows ?? [];
139
+ const normalizedRows = rows
140
+ .map((row) => (row.tableCells ?? []).map((cell) => normalizeTableCell(cell, lists)))
141
+ .filter((cells) => cells.length > 0);
142
+ if (normalizedRows.length === 0) {
143
+ return [];
144
+ }
145
+ const columnCount = Math.max(...normalizedRows.map((cells) => cells.length));
146
+ const paddedRows = normalizedRows.map((cells) => Array.from({ length: columnCount }, (_, index) => cells[index] ?? ''));
147
+ const [header, ...body] = paddedRows;
148
+ const blocks = [`| ${header.join(' | ')} |`, markdownTableDivider(columnCount)];
149
+ for (const row of body) {
150
+ blocks.push(`| ${row.join(' | ')} |`);
151
+ }
152
+ return [blocks.join('\n')];
153
+ }
154
+ function normalizeStructuralElements(elements, lists) {
155
+ const blocks = [];
156
+ for (const element of elements) {
157
+ const line = paragraphToMarkdown(element.paragraph, lists);
158
+ if (line) {
159
+ blocks.push(line);
160
+ continue;
161
+ }
162
+ if (element.table) {
163
+ blocks.push(...normalizeTable(element.table, lists));
164
+ }
165
+ }
166
+ return blocks;
167
+ }
168
+ function headerFooterRoleMap(label, documentStyle) {
169
+ const roleMap = new Map();
170
+ const roleEntries = label === 'Headers'
171
+ ? [
172
+ [documentStyle?.defaultHeaderId, 'Default Header'],
173
+ [documentStyle?.firstPageHeaderId, 'First Page Header'],
174
+ [documentStyle?.evenPageHeaderId, 'Even Page Header'],
175
+ ]
176
+ : [
177
+ [documentStyle?.defaultFooterId, 'Default Footer'],
178
+ [documentStyle?.firstPageFooterId, 'First Page Footer'],
179
+ [documentStyle?.evenPageFooterId, 'Even Page Footer'],
180
+ ];
181
+ for (const [id, role] of roleEntries) {
182
+ const normalizedId = id?.trim();
183
+ if (!normalizedId || roleMap.has(normalizedId)) {
184
+ continue;
185
+ }
186
+ roleMap.set(normalizedId, role ?? normalizedId);
187
+ }
188
+ return roleMap;
189
+ }
190
+ function normalizeHeaderFooterMap(label, entries, lists, documentStyle) {
191
+ if (!entries) {
192
+ return null;
193
+ }
194
+ const ids = Object.keys(entries).sort();
195
+ const roles = headerFooterRoleMap(label, documentStyle);
196
+ const sections = [];
197
+ for (const id of ids) {
198
+ const blocks = normalizeStructuralElements(entries[id]?.content ?? [], lists);
199
+ if (blocks.length === 0) {
200
+ continue;
201
+ }
202
+ const title = roles.get(id) ?? `${label.slice(0, -1)} ${escapeMarkdownText(id)}`;
203
+ sections.push(`### ${title}\n\n${blocks.join('\n\n').trim()}`);
204
+ }
205
+ if (sections.length === 0) {
206
+ return null;
207
+ }
208
+ return `## ${label}\n\n${sections.join('\n\n').trim()}`;
209
+ }
210
+ function joinNonEmptySections(sections) {
211
+ const nonEmpty = sections.filter((section) => Boolean(section?.trim()));
212
+ if (nonEmpty.length === 0) {
213
+ return null;
214
+ }
215
+ return nonEmpty.join('\n\n').trim();
216
+ }
217
+ function flattenGoogleDocsTabs(tabs) {
218
+ if (!tabs?.length) {
219
+ return [];
220
+ }
221
+ const flattened = [];
222
+ for (const tab of tabs) {
223
+ flattened.push(tab);
224
+ flattened.push(...flattenGoogleDocsTabs(tab.childTabs));
225
+ }
226
+ return flattened;
227
+ }
228
+ function normalizeTab(tab, fallbackLists) {
229
+ const lists = tab.documentTab?.lists ?? fallbackLists;
230
+ const headerSection = normalizeHeaderFooterMap('Headers', tab.documentTab?.headers, lists, tab.documentTab?.documentStyle);
231
+ const bodySection = normalizeStructuralElements(tab.documentTab?.body?.content ?? [], lists).join('\n\n').trim();
232
+ const footerSection = normalizeHeaderFooterMap('Footers', tab.documentTab?.footers, lists, tab.documentTab?.documentStyle);
233
+ const content = joinNonEmptySections([headerSection, bodySection, footerSection]);
234
+ if (!content) {
235
+ return null;
236
+ }
237
+ const title = tab.tabProperties?.title?.trim();
238
+ if (!title) {
239
+ return content;
240
+ }
241
+ return [`# ${escapeMarkdownText(title)}`, content].join('\n\n').trim();
242
+ }
243
+ export function normalizeGoogleDocToMarkdown(document) {
244
+ const normalizedTabs = flattenGoogleDocsTabs(document.tabs)
245
+ .map((tab) => normalizeTab(tab, document.lists))
246
+ .filter((tab) => Boolean(tab));
247
+ if (normalizedTabs.length > 0) {
248
+ return normalizedTabs.join('\n\n').trim();
249
+ }
250
+ const bodySection = normalizeStructuralElements(document.body?.content ?? [], document.lists).join('\n\n').trim();
251
+ return (joinNonEmptySections([
252
+ normalizeHeaderFooterMap('Headers', document.headers, document.lists, document.documentStyle),
253
+ bodySection,
254
+ normalizeHeaderFooterMap('Footers', document.footers, document.lists, document.documentStyle),
255
+ ]) ?? '');
256
+ }
@@ -0,0 +1,153 @@
1
+ import { z } from 'zod';
2
+ export declare const GDRIVE_SCOPES: readonly ["https://www.googleapis.com/auth/drive.readonly", "https://www.googleapis.com/auth/documents.readonly"];
3
+ export declare const GDRIVE_SOURCE_KEY = "gdrive";
4
+ export declare const GDRIVE_DOC_MIME_TYPE = "application/vnd.google-apps.document";
5
+ export declare const GDRIVE_FOLDER_MIME_TYPE = "application/vnd.google-apps.folder";
6
+ export declare const gdrivePullConfigSchema: z.ZodObject<{
7
+ serviceAccountKey: z.ZodString;
8
+ folderId: z.ZodString;
9
+ recursive: z.ZodDefault<z.ZodBoolean>;
10
+ }, z.core.$strip>;
11
+ export type GdrivePullConfig = z.infer<typeof gdrivePullConfigSchema>;
12
+ export declare const gdriveManifestSchema: z.ZodObject<{
13
+ source: z.ZodLiteral<"gdrive">;
14
+ folderId: z.ZodString;
15
+ recursive: z.ZodBoolean;
16
+ fetchedAt: z.ZodString;
17
+ fileCount: z.ZodNumber;
18
+ skipped: z.ZodDefault<z.ZodArray<z.ZodObject<{
19
+ externalId: z.ZodString;
20
+ reason: z.ZodString;
21
+ }, z.core.$strip>>>;
22
+ warnings: z.ZodDefault<z.ZodArray<z.ZodString>>;
23
+ }, z.core.$strip>;
24
+ export type GdriveManifest = z.infer<typeof gdriveManifestSchema>;
25
+ export declare const gdriveMetadataSchema: z.ZodObject<{
26
+ id: z.ZodString;
27
+ title: z.ZodString;
28
+ path: z.ZodString;
29
+ url: z.ZodDefault<z.ZodNullable<z.ZodString>>;
30
+ mimeType: z.ZodLiteral<"application/vnd.google-apps.document">;
31
+ folderId: z.ZodString;
32
+ drivePath: z.ZodDefault<z.ZodArray<z.ZodString>>;
33
+ modifiedTime: z.ZodDefault<z.ZodNullable<z.ZodString>>;
34
+ }, z.core.$strip>;
35
+ export declare const gdriveServiceAccountKeySchema: z.ZodObject<{
36
+ client_email: z.ZodString;
37
+ private_key: z.ZodString;
38
+ project_id: z.ZodOptional<z.ZodString>;
39
+ }, z.core.$strip>;
40
+ export type GdriveServiceAccountKey = z.infer<typeof gdriveServiceAccountKeySchema>;
41
+ export interface GdriveFileRecord {
42
+ id: string;
43
+ name: string;
44
+ mimeType: string;
45
+ parents: string[];
46
+ webViewLink: string | null;
47
+ modifiedTime: string | null;
48
+ }
49
+ export interface GoogleDocsDocument {
50
+ documentId?: string;
51
+ title?: string;
52
+ body?: {
53
+ content?: GoogleDocsStructuralElement[];
54
+ };
55
+ documentStyle?: GoogleDocsDocumentStyle;
56
+ lists?: Record<string, GoogleDocsList>;
57
+ headers?: Record<string, GoogleDocsHeaderFooter>;
58
+ footers?: Record<string, GoogleDocsHeaderFooter>;
59
+ tabs?: GoogleDocsTab[];
60
+ }
61
+ export interface GoogleDocsList {
62
+ listProperties?: {
63
+ nestingLevels?: GoogleDocsListNestingLevel[];
64
+ };
65
+ }
66
+ interface GoogleDocsListNestingLevel {
67
+ glyphType?: string;
68
+ glyphSymbol?: string;
69
+ }
70
+ export interface GoogleDocsTab {
71
+ tabProperties?: {
72
+ tabId?: string;
73
+ title?: string;
74
+ };
75
+ childTabs?: GoogleDocsTab[];
76
+ documentTab?: {
77
+ body?: {
78
+ content?: GoogleDocsStructuralElement[];
79
+ };
80
+ documentStyle?: GoogleDocsDocumentStyle;
81
+ lists?: Record<string, GoogleDocsList>;
82
+ headers?: Record<string, GoogleDocsHeaderFooter>;
83
+ footers?: Record<string, GoogleDocsHeaderFooter>;
84
+ };
85
+ }
86
+ export interface GoogleDocsDocumentStyle {
87
+ defaultHeaderId?: string;
88
+ defaultFooterId?: string;
89
+ firstPageHeaderId?: string;
90
+ firstPageFooterId?: string;
91
+ evenPageHeaderId?: string;
92
+ evenPageFooterId?: string;
93
+ }
94
+ export interface GoogleDocsHeaderFooter {
95
+ headerId?: string;
96
+ footerId?: string;
97
+ content?: GoogleDocsStructuralElement[];
98
+ }
99
+ export interface GoogleDocsStructuralElement {
100
+ paragraph?: GoogleDocsParagraph;
101
+ table?: GoogleDocsTable;
102
+ sectionBreak?: unknown;
103
+ }
104
+ export interface GoogleDocsTable {
105
+ tableRows?: GoogleDocsTableRow[];
106
+ }
107
+ interface GoogleDocsTableRow {
108
+ tableCells?: GoogleDocsTableCell[];
109
+ }
110
+ export interface GoogleDocsTableCell {
111
+ content?: GoogleDocsStructuralElement[];
112
+ }
113
+ export interface GoogleDocsParagraph {
114
+ elements?: GoogleDocsParagraphElement[];
115
+ bullet?: {
116
+ listId?: string;
117
+ nestingLevel?: number;
118
+ };
119
+ paragraphStyle?: {
120
+ namedStyleType?: string;
121
+ headingId?: string;
122
+ };
123
+ }
124
+ export interface GoogleDocsLinkTarget {
125
+ id?: string;
126
+ tabId?: string;
127
+ }
128
+ export interface GoogleDocsParagraphElement {
129
+ textRun?: {
130
+ content?: string;
131
+ textStyle?: {
132
+ bold?: boolean;
133
+ italic?: boolean;
134
+ underline?: boolean;
135
+ strikethrough?: boolean;
136
+ link?: {
137
+ url?: string;
138
+ tabId?: string;
139
+ headingId?: string;
140
+ bookmarkId?: string;
141
+ heading?: GoogleDocsLinkTarget;
142
+ bookmark?: GoogleDocsLinkTarget;
143
+ };
144
+ weightedFontFamily?: {
145
+ fontFamily?: string;
146
+ };
147
+ baselineOffset?: 'SUPERSCRIPT' | 'SUBSCRIPT' | string;
148
+ };
149
+ };
150
+ inlineObjectElement?: unknown;
151
+ pageBreak?: unknown;
152
+ }
153
+ export {};
@@ -0,0 +1,36 @@
1
+ import { z } from 'zod';
2
+ const GDRIVE_DOCS_SCOPE = 'https://www.googleapis.com/auth/documents.readonly';
3
+ const GDRIVE_DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.readonly';
4
+ export const GDRIVE_SCOPES = [GDRIVE_DRIVE_SCOPE, GDRIVE_DOCS_SCOPE];
5
+ export const GDRIVE_SOURCE_KEY = 'gdrive';
6
+ export const GDRIVE_DOC_MIME_TYPE = 'application/vnd.google-apps.document';
7
+ export const GDRIVE_FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder';
8
+ export const gdrivePullConfigSchema = z.object({
9
+ serviceAccountKey: z.string().min(1),
10
+ folderId: z.string().min(1),
11
+ recursive: z.boolean().default(false),
12
+ });
13
+ export const gdriveManifestSchema = z.object({
14
+ source: z.literal(GDRIVE_SOURCE_KEY),
15
+ folderId: z.string().min(1),
16
+ recursive: z.boolean(),
17
+ fetchedAt: z.string().datetime(),
18
+ fileCount: z.number().int().nonnegative(),
19
+ skipped: z.array(z.object({ externalId: z.string(), reason: z.string() })).default([]),
20
+ warnings: z.array(z.string()).default([]),
21
+ });
22
+ export const gdriveMetadataSchema = z.object({
23
+ id: z.string(),
24
+ title: z.string(),
25
+ path: z.string(),
26
+ url: z.string().nullable().default(null),
27
+ mimeType: z.literal(GDRIVE_DOC_MIME_TYPE),
28
+ folderId: z.string(),
29
+ drivePath: z.array(z.string()).default([]),
30
+ modifiedTime: z.string().datetime().nullable().default(null),
31
+ });
32
+ export const gdriveServiceAccountKeySchema = z.object({
33
+ client_email: z.string().email(),
34
+ private_key: z.string().min(1),
35
+ project_id: z.string().min(1).optional(),
36
+ });
@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process';
2
2
  import { request as httpRequest } from 'node:http';
3
3
  import { request as httpsRequest } from 'node:https';
4
4
  import { URL } from 'node:url';
5
+ import { isKtxScanWarningCode } from '../../../scan/local-structural-artifacts.js';
5
6
  import { tableRefFromKey } from '../../../scan/table-ref.js';
6
7
  import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
7
8
  const DEFAULT_SCHEMAS = ['public'];
@@ -147,7 +148,29 @@ function mapTable(raw) {
147
148
  foreignKeys: recordArray(raw.foreign_keys).map(mapForeignKey),
148
149
  };
149
150
  }
151
+ function mapWarning(raw) {
152
+ const code = optionalString(raw.code);
153
+ // Drop codes Node cannot render, keeping the daemon and Node warning catalogs
154
+ // in parity rather than surfacing an unknown code downstream.
155
+ if (!code || !isKtxScanWarningCode(code))
156
+ return null;
157
+ const table = optionalString(raw.table);
158
+ const column = optionalString(raw.column);
159
+ return {
160
+ code,
161
+ message: requiredString(raw.message, 'warnings[].message'),
162
+ recoverable: raw.recoverable !== false,
163
+ ...(table ? { table } : {}),
164
+ ...(column ? { column } : {}),
165
+ ...(raw.metadata && typeof raw.metadata === 'object' && !Array.isArray(raw.metadata)
166
+ ? { metadata: recordValue(raw.metadata) }
167
+ : {}),
168
+ };
169
+ }
150
170
  function mapDaemonSnapshot(raw, input) {
171
+ const warnings = recordArray(raw.warnings)
172
+ .map(mapWarning)
173
+ .filter((warning) => warning !== null);
151
174
  return {
152
175
  connectionId: requiredString(raw.connection_id, 'connection_id') || input.connectionId,
153
176
  driver: 'postgres',
@@ -155,6 +178,7 @@ function mapDaemonSnapshot(raw, input) {
155
178
  scope: { schemas: input.schemas },
156
179
  metadata: recordValue(raw.metadata),
157
180
  tables: recordArray(raw.tables).map(mapTable),
181
+ ...(warnings.length > 0 ? { warnings } : {}),
158
182
  };
159
183
  }
160
184
  function serializeTableScope(options) {
@@ -0,0 +1,8 @@
1
+ import type { SourceFetchReport } from '../../types.js';
2
+ /**
3
+ * Derives the fetch report from the staged `warnings.json`: objects that failed
4
+ * introspection become `skipped` entries so the run report, ingest summary, and
5
+ * `ktx status` can surface them. Returns null when nothing was skipped, keeping
6
+ * clean ingests free of an empty report.
7
+ */
8
+ export declare function readLiveDatabaseFetchReport(stagedDir: string): Promise<SourceFetchReport | null>;
@@ -0,0 +1,37 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+ import { LIVE_DATABASE_WARNINGS_FILE } from './stage.js';
4
+ const OBJECT_SKIP_CODE = 'object_introspection_failed';
5
+ /**
6
+ * Derives the fetch report from the staged `warnings.json`: objects that failed
7
+ * introspection become `skipped` entries so the run report, ingest summary, and
8
+ * `ktx status` can surface them. Returns null when nothing was skipped, keeping
9
+ * clean ingests free of an empty report.
10
+ */
11
+ export async function readLiveDatabaseFetchReport(stagedDir) {
12
+ let parsed;
13
+ try {
14
+ parsed = JSON.parse(await readFile(join(stagedDir, LIVE_DATABASE_WARNINGS_FILE), 'utf8'));
15
+ }
16
+ catch {
17
+ return null;
18
+ }
19
+ const warnings = parsed && typeof parsed === 'object' && Array.isArray(parsed.warnings)
20
+ ? (parsed.warnings)
21
+ : [];
22
+ const skipped = warnings
23
+ .filter((warning) => warning.code === OBJECT_SKIP_CODE)
24
+ .map((warning) => ({
25
+ rawPath: '',
26
+ entityType: 'database_object',
27
+ entityId: typeof warning.table === 'string' ? warning.table : null,
28
+ severity: 'warning',
29
+ statusCode: null,
30
+ message: typeof warning.message === 'string' ? warning.message : 'introspection failed',
31
+ retryRecommended: false,
32
+ }));
33
+ if (skipped.length === 0) {
34
+ return null;
35
+ }
36
+ return { status: 'partial', retryRecommended: false, skipped, warnings: [] };
37
+ }
@@ -1,4 +1,4 @@
1
- import type { ChunkResult, DiffSet, FetchContext, SourceAdapter } from '../../types.js';
1
+ import type { ChunkResult, DiffSet, FetchContext, SourceAdapter, SourceFetchReport } from '../../types.js';
2
2
  import type { LiveDatabaseSourceAdapterDeps } from './types.js';
3
3
  export declare class LiveDatabaseSourceAdapter implements SourceAdapter {
4
4
  private readonly deps;
@@ -6,6 +6,7 @@ export declare class LiveDatabaseSourceAdapter implements SourceAdapter {
6
6
  readonly skillNames: string[];
7
7
  constructor(deps: LiveDatabaseSourceAdapterDeps);
8
8
  detect(stagedDir: string): Promise<boolean>;
9
+ readFetchReport(stagedDir: string): Promise<SourceFetchReport | null>;
9
10
  fetch(_pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise<void>;
10
11
  chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
11
12
  }
@@ -1,4 +1,6 @@
1
1
  import { chunkLiveDatabaseStagedDir } from './chunk.js';
2
+ import { readLiveDatabaseFetchReport } from './fetch-report.js';
3
+ import { assertLiveDatabaseScanOutcome } from './scan-outcome.js';
2
4
  import { detectLiveDatabaseStagedDir, writeLiveDatabaseSnapshot } from './stage.js';
3
5
  export class LiveDatabaseSourceAdapter {
4
6
  deps;
@@ -10,14 +12,19 @@ export class LiveDatabaseSourceAdapter {
10
12
  detect(stagedDir) {
11
13
  return detectLiveDatabaseStagedDir(stagedDir);
12
14
  }
15
+ readFetchReport(stagedDir) {
16
+ return readLiveDatabaseFetchReport(stagedDir);
17
+ }
13
18
  async fetch(_pullConfig, stagedDir, ctx) {
14
19
  const tableScope = ctx.tableScope;
15
20
  const snapshot = await this.deps.introspection.extractSchema(ctx.connectionId, { tableScope });
16
- await writeLiveDatabaseSnapshot(stagedDir, {
21
+ const finalized = {
17
22
  ...snapshot,
18
23
  connectionId: ctx.connectionId,
19
24
  extractedAt: snapshot.extractedAt ?? (this.deps.now ?? (() => new Date()))().toISOString(),
20
- });
25
+ };
26
+ assertLiveDatabaseScanOutcome({ connectionId: ctx.connectionId, scope: tableScope, snapshot: finalized });
27
+ await writeLiveDatabaseSnapshot(stagedDir, finalized);
21
28
  }
22
29
  chunk(stagedDir, diffSet) {
23
30
  return chunkLiveDatabaseStagedDir(stagedDir, diffSet);
@@ -64,6 +64,8 @@ export interface BuildLiveDatabaseManifestShardsResult {
64
64
  }
65
65
  export declare function mergeUsagePreservingExternal(existing: TableUsageOutput | undefined, incoming: TableUsageOutput | undefined): TableUsageOutput | undefined;
66
66
  /** @internal */
67
+ export declare function buildTableRef(name: string, catalog: string | null, db: string | null): string;
68
+ /** @internal */
67
69
  export declare function buildJoinsByTable(tableNames: Set<string>, joins: LiveDatabaseManifestJoinData[], preservedJoins: Map<string, LiveDatabaseManifestJoinEntry[]>, federatedSiblingTargets?: Set<string>): Map<string, LiveDatabaseManifestJoinEntry[]>;
68
70
  export declare function buildLiveDatabaseManifestShards(input: BuildLiveDatabaseManifestShardsInput): BuildLiveDatabaseManifestShardsResult;
69
71
  export {};
@@ -73,7 +73,8 @@ function getShardKey(connectionType, catalog, db) {
73
73
  }
74
74
  }
75
75
  }
76
- function buildTableRef(name, catalog, db) {
76
+ /** @internal */
77
+ export function buildTableRef(name, catalog, db) {
77
78
  const parts = [];
78
79
  if (catalog) {
79
80
  parts.push(catalog);
@@ -154,7 +155,10 @@ export function buildLiveDatabaseManifestShards(input) {
154
155
  for (const table of input.tables) {
155
156
  const shardKey = getShardKey(input.connectionType, table.catalog, table.db);
156
157
  const shard = shards.get(shardKey) ?? { tables: {} };
157
- const existingDescriptions = input.existingDescriptions?.get(table.name);
158
+ // Existing descriptions/usage are keyed by the fully-qualified ref so two
159
+ // same-named tables in different schemas never share an entry.
160
+ const fullRef = buildTableRef(table.name, table.catalog, table.db);
161
+ const existingDescriptions = input.existingDescriptions?.get(fullRef);
158
162
  const columns = table.columns.map((column) => {
159
163
  const manifestColumn = {
160
164
  name: column.name,
@@ -173,14 +177,14 @@ export function buildLiveDatabaseManifestShards(input) {
173
177
  return manifestColumn;
174
178
  });
175
179
  const entry = {
176
- table: buildTableRef(table.name, table.catalog, table.db),
180
+ table: fullRef,
177
181
  columns,
178
182
  };
179
183
  const tableDescriptions = mergeDescriptionsPreservingExternal(existingDescriptions?.table, table.descriptions);
180
184
  if (tableDescriptions) {
181
185
  entry.descriptions = tableDescriptions;
182
186
  }
183
- const usage = mergeUsagePreservingExternal(input.existingUsage?.get(table.name), table.usage);
187
+ const usage = mergeUsagePreservingExternal(input.existingUsage?.get(fullRef), table.usage);
184
188
  if (usage) {
185
189
  entry.usage = usage;
186
190
  }
@@ -0,0 +1,17 @@
1
+ import { type KtxTableRefKey } from '../../../scan/table-ref.js';
2
+ import type { KtxSchemaSnapshot } from '../../../scan/types.js';
3
+ /**
4
+ * Enforces the partial-vs-total outcome rules for a live-database snapshot,
5
+ * uniformly for every connector. Outcomes follow from object counts, not a
6
+ * mode: a connection with at least one ingested object succeeds (any broken
7
+ * objects ride along as warnings); a connection where every introspected object
8
+ * failed, or a non-empty enabled_tables scope that matched nothing, raises a
9
+ * clear connection error instead of staging an empty layer that would later
10
+ * surface as the generic "did not recognize" message. A legitimately empty
11
+ * database (no scope, no objects) succeeds with an empty layer.
12
+ */
13
+ export declare function assertLiveDatabaseScanOutcome(input: {
14
+ connectionId: string;
15
+ scope: ReadonlySet<KtxTableRefKey> | undefined;
16
+ snapshot: KtxSchemaSnapshot;
17
+ }): void;