@kaelio/ktx 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.13.0-py3-none-any.whl → kaelio_ktx-0.14.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +1 -1
- package/dist/commands/ingest-commands.d.ts +9 -0
- package/dist/commands/ingest-commands.js +37 -1
- package/dist/commands/knowledge-commands.js +3 -0
- package/dist/commands/setup-commands.js +14 -1
- package/dist/connection-drivers.d.ts +2 -0
- package/dist/connection-drivers.js +7 -3
- package/dist/connection.d.ts +3 -0
- package/dist/connection.js +27 -0
- package/dist/connectors/bigquery/connector.d.ts +17 -6
- package/dist/connectors/bigquery/connector.js +152 -76
- package/dist/connectors/bigquery/dialect.d.ts +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +1 -0
- package/dist/connectors/clickhouse/connector.js +45 -16
- package/dist/connectors/clickhouse/dialect.d.ts +2 -2
- package/dist/connectors/mongodb/connector.d.ts +69 -0
- package/dist/connectors/mongodb/connector.js +307 -0
- package/dist/connectors/mongodb/dialect.d.ts +17 -0
- package/dist/connectors/mongodb/dialect.js +50 -0
- package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
- package/dist/connectors/mongodb/live-database-introspection.js +24 -0
- package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
- package/dist/connectors/mongodb/schema-inference.js +110 -0
- package/dist/connectors/mysql/connector.d.ts +1 -0
- package/dist/connectors/mysql/connector.js +18 -2
- package/dist/connectors/mysql/dialect.d.ts +2 -2
- package/dist/connectors/postgres/connector.d.ts +1 -0
- package/dist/connectors/postgres/connector.js +20 -3
- package/dist/connectors/postgres/dialect.d.ts +2 -2
- package/dist/connectors/snowflake/connector.d.ts +1 -0
- package/dist/connectors/snowflake/connector.js +27 -3
- package/dist/connectors/snowflake/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/connector.d.ts +11 -1
- package/dist/connectors/sqlite/connector.js +120 -17
- package/dist/connectors/sqlite/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
- package/dist/connectors/sqlite/read-query-child.js +23 -0
- package/dist/connectors/sqlserver/connector.d.ts +2 -0
- package/dist/connectors/sqlserver/connector.js +23 -5
- package/dist/connectors/sqlserver/dialect.d.ts +2 -2
- package/dist/context/cache/content-result-cache.d.ts +36 -0
- package/dist/context/cache/content-result-cache.js +14 -0
- package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
- package/dist/context/cache/sqlite-content-result-cache.js +223 -0
- package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
- package/dist/context/connections/bigquery-identifiers.js +7 -0
- package/dist/context/connections/configured-connections.d.ts +9 -0
- package/dist/context/connections/configured-connections.js +18 -0
- package/dist/context/connections/dialects.d.ts +30 -4
- package/dist/context/connections/dialects.js +38 -11
- package/dist/context/connections/drivers.js +21 -0
- package/dist/context/connections/gdrive-config.d.ts +19 -0
- package/dist/context/connections/gdrive-config.js +57 -0
- package/dist/context/connections/query-deadline.d.ts +31 -0
- package/dist/context/connections/query-deadline.js +37 -0
- package/dist/context/connections/read-only-sql.d.ts +5 -0
- package/dist/context/connections/read-only-sql.js +139 -1
- package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
- package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
- package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
- package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
- package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
- package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
- package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
- package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
- package/dist/context/ingest/adapters/gdrive/types.js +36 -0
- package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
- package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
- package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
- package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
- package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
- package/dist/context/ingest/adapters/live-database/stage.js +5 -5
- package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/types.js +1 -1
- package/dist/context/ingest/artifact-gates.d.ts +32 -1
- package/dist/context/ingest/artifact-gates.js +85 -18
- package/dist/context/ingest/final-gate-prune.d.ts +35 -0
- package/dist/context/ingest/final-gate-prune.js +229 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +459 -240
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
- package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
- package/dist/context/ingest/local-adapters.js +6 -0
- package/dist/context/ingest/local-bundle-runtime.js +7 -2
- package/dist/context/ingest/local-ingest.js +1 -1
- package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
- package/dist/context/ingest/local-stage-ingest.js +3 -1
- package/dist/context/ingest/ports.d.ts +3 -0
- package/dist/context/ingest/report-snapshot.js +13 -3
- package/dist/context/ingest/reports.d.ts +3 -3
- package/dist/context/ingest/reports.js +3 -1
- package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
- package/dist/context/ingest/stages/build-wu-context.js +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
- package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
- package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
- package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
- package/dist/context/ingest/types.d.ts +3 -0
- package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
- package/dist/context/ingest/wiki-body-refs.js +37 -8
- package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
- package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
- package/dist/context/ingest/work-unit-cache.d.ts +67 -0
- package/dist/context/ingest/work-unit-cache.js +230 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.js +5 -0
- package/dist/context/llm/claude-code-runtime.d.ts +4 -1
- package/dist/context/llm/claude-code-runtime.js +38 -8
- package/dist/context/llm/codex-runtime.d.ts +4 -1
- package/dist/context/llm/codex-runtime.js +48 -35
- package/dist/context/llm/runtime-port.d.ts +26 -0
- package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
- package/dist/context/llm/subprocess-generate-object-child.js +34 -0
- package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
- package/dist/context/llm/subprocess-generate-object.js +122 -0
- package/dist/context/mcp/context-tools.d.ts +2 -0
- package/dist/context/mcp/context-tools.js +111 -17
- package/dist/context/mcp/local-project-ports.d.ts +7 -0
- package/dist/context/mcp/local-project-ports.js +29 -0
- package/dist/context/mcp/logger.d.ts +24 -0
- package/dist/context/mcp/logger.js +49 -0
- package/dist/context/mcp/server.js +2 -0
- package/dist/context/mcp/types.d.ts +17 -0
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +42 -0
- package/dist/context/project/config.js +5 -0
- package/dist/context/project/driver-schemas.d.ts +20 -0
- package/dist/context/project/driver-schemas.js +50 -1
- package/dist/context/project/project.js +1 -1
- package/dist/context/project/setup-config.js +1 -0
- package/dist/context/scan/description-generation.d.ts +4 -0
- package/dist/context/scan/description-generation.js +100 -13
- package/dist/context/scan/enabled-tables.d.ts +5 -0
- package/dist/context/scan/enabled-tables.js +13 -1
- package/dist/context/scan/enrichment-state.d.ts +50 -7
- package/dist/context/scan/enrichment-state.js +30 -13
- package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
- package/dist/context/scan/local-enrichment-artifacts.js +155 -32
- package/dist/context/scan/local-enrichment.d.ts +39 -4
- package/dist/context/scan/local-enrichment.js +345 -92
- package/dist/context/scan/local-scan.d.ts +4 -1
- package/dist/context/scan/local-scan.js +54 -13
- package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
- package/dist/context/scan/local-structural-artifacts.js +5 -0
- package/dist/context/scan/object-introspection.d.ts +21 -0
- package/dist/context/scan/object-introspection.js +35 -0
- package/dist/context/scan/relationship-benchmarks.js +5 -3
- package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
- package/dist/context/scan/relationship-composite-candidates.js +13 -1
- package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
- package/dist/context/scan/relationship-detection-budget.js +53 -0
- package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
- package/dist/context/scan/relationship-diagnostics.js +2 -0
- package/dist/context/scan/relationship-discovery.d.ts +9 -3
- package/dist/context/scan/relationship-discovery.js +27 -4
- package/dist/context/scan/relationship-llm-proposal.js +36 -27
- package/dist/context/scan/relationship-profiling.d.ts +7 -3
- package/dist/context/scan/relationship-profiling.js +53 -41
- package/dist/context/scan/relationship-validation.d.ts +6 -4
- package/dist/context/scan/relationship-validation.js +46 -32
- package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
- package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
- package/dist/context/scan/types.d.ts +5 -3
- package/dist/context/scan/types.js +2 -0
- package/dist/context/sl/local-query.js +5 -0
- package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
- package/dist/context/sl/semantic-layer.service.js +1 -1
- package/dist/context/sl/source-files.d.ts +5 -0
- package/dist/context/sl/source-files.js +17 -11
- package/dist/context/sl/tools/connection-id-schema.js +1 -1
- package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
- package/dist/context/sql-analysis/dialect-notes.js +40 -0
- package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
- package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
- package/dist/context/sql-analysis/dialects/mysql.md +9 -0
- package/dist/context/sql-analysis/dialects/postgres.md +10 -0
- package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
- package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
- package/dist/context/sql-analysis/dialects/tsql.md +10 -0
- package/dist/context/wiki/keys.js +1 -1
- package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
- package/dist/context/wiki/knowledge-wiki.service.js +2 -1
- package/dist/context/wiki/local-knowledge.d.ts +13 -0
- package/dist/context/wiki/local-knowledge.js +50 -6
- package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
- package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
- package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
- package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
- package/dist/context/wiki/types.d.ts +6 -0
- package/dist/context-build-view.d.ts +3 -0
- package/dist/context-build-view.js +1 -0
- package/dist/knowledge.d.ts +2 -0
- package/dist/knowledge.js +12 -1
- package/dist/local-adapters.js +11 -0
- package/dist/mcp-http-server.js +15 -1
- package/dist/mcp-server-factory.d.ts +2 -0
- package/dist/mcp-server-factory.js +15 -1
- package/dist/mcp-stdio-server.js +10 -2
- package/dist/notion-page-picker.js +1 -1
- package/dist/prompts/memory_agent_external_ingest.md +2 -0
- package/dist/public-ingest.d.ts +3 -1
- package/dist/public-ingest.js +3 -0
- package/dist/scan.d.ts +3 -1
- package/dist/scan.js +7 -0
- package/dist/setup-databases.d.ts +1 -1
- package/dist/setup-databases.js +43 -1
- package/dist/setup-sources.d.ts +5 -1
- package/dist/setup-sources.js +124 -48
- package/dist/setup.d.ts +3 -0
- package/dist/setup.js +6 -1
- package/dist/skills/analytics/SKILL.md +200 -2
- package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
- package/dist/skills/wiki_capture/SKILL.md +24 -0
- package/dist/sql.js +4 -0
- package/dist/status-project.d.ts +4 -0
- package/dist/status-project.js +54 -2
- package/dist/telemetry/events.d.ts +2 -2
- package/dist/text-ingest.d.ts +4 -0
- package/dist/text-ingest.js +58 -29
- package/dist/verbatim-ingest.d.ts +46 -0
- package/dist/verbatim-ingest.js +193 -0
- package/package.json +5 -1
- package/dist/context/ingest/final-gate-repair.d.ts +0 -28
- package/dist/context/ingest/final-gate-repair.js +0 -91
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
function escapeMarkdownText(value) {
|
|
2
|
+
return value.replace(/([*_~`])/g, '\\$1');
|
|
3
|
+
}
|
|
4
|
+
function normalizeInternalLinkTarget(prefix, target) {
|
|
5
|
+
const id = typeof target === 'string' ? target : target?.id;
|
|
6
|
+
if (!id?.trim()) {
|
|
7
|
+
return null;
|
|
8
|
+
}
|
|
9
|
+
return `#${prefix}-${id.trim()}`;
|
|
10
|
+
}
|
|
11
|
+
function resolveLinkHref(element) {
|
|
12
|
+
const link = element.textRun?.textStyle?.link;
|
|
13
|
+
const href = link?.url?.trim();
|
|
14
|
+
if (href) {
|
|
15
|
+
return href;
|
|
16
|
+
}
|
|
17
|
+
return (normalizeInternalLinkTarget('heading', link?.heading) ??
|
|
18
|
+
normalizeInternalLinkTarget('heading', link?.headingId) ??
|
|
19
|
+
normalizeInternalLinkTarget('bookmark', link?.bookmark) ??
|
|
20
|
+
normalizeInternalLinkTarget('bookmark', link?.bookmarkId) ??
|
|
21
|
+
null);
|
|
22
|
+
}
|
|
23
|
+
function normalizeTextRun(element) {
|
|
24
|
+
const content = element.textRun?.content ?? '';
|
|
25
|
+
const style = element.textRun?.textStyle;
|
|
26
|
+
let text = escapeMarkdownText(content.replace(/\r/g, ''));
|
|
27
|
+
if (!text && element.inlineObjectElement) {
|
|
28
|
+
return '[Embedded object]';
|
|
29
|
+
}
|
|
30
|
+
if (!text && element.pageBreak) {
|
|
31
|
+
return '\n---\n';
|
|
32
|
+
}
|
|
33
|
+
if (!text) {
|
|
34
|
+
return '';
|
|
35
|
+
}
|
|
36
|
+
const href = resolveLinkHref(element);
|
|
37
|
+
const isCode = style?.weightedFontFamily?.fontFamily === 'Courier New';
|
|
38
|
+
if (isCode) {
|
|
39
|
+
text = `\`${text.replace(/`/g, '\\`')}\``;
|
|
40
|
+
}
|
|
41
|
+
if (style?.bold) {
|
|
42
|
+
text = `**${text}**`;
|
|
43
|
+
}
|
|
44
|
+
if (style?.italic) {
|
|
45
|
+
text = `*${text}*`;
|
|
46
|
+
}
|
|
47
|
+
if (style?.underline) {
|
|
48
|
+
text = `<u>${text}</u>`;
|
|
49
|
+
}
|
|
50
|
+
if (style?.strikethrough) {
|
|
51
|
+
text = `~~${text}~~`;
|
|
52
|
+
}
|
|
53
|
+
if (href) {
|
|
54
|
+
text = `[${text}](${href.replace(/\)/g, '\\)')})`;
|
|
55
|
+
}
|
|
56
|
+
if (style?.baselineOffset === 'SUPERSCRIPT') {
|
|
57
|
+
text = `<sup>${text}</sup>`;
|
|
58
|
+
}
|
|
59
|
+
else if (style?.baselineOffset === 'SUBSCRIPT') {
|
|
60
|
+
text = `<sub>${text}</sub>`;
|
|
61
|
+
}
|
|
62
|
+
return text;
|
|
63
|
+
}
|
|
64
|
+
function paragraphText(paragraph) {
|
|
65
|
+
return (paragraph?.elements ?? [])
|
|
66
|
+
.map((element) => normalizeTextRun(element))
|
|
67
|
+
.join('')
|
|
68
|
+
.replace(/\n/g, '')
|
|
69
|
+
.trim();
|
|
70
|
+
}
|
|
71
|
+
function headingPrefix(namedStyleType) {
|
|
72
|
+
if (namedStyleType === 'TITLE') {
|
|
73
|
+
return '#';
|
|
74
|
+
}
|
|
75
|
+
if (namedStyleType === 'SUBTITLE') {
|
|
76
|
+
return '##';
|
|
77
|
+
}
|
|
78
|
+
if (!namedStyleType?.startsWith('HEADING_')) {
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
const level = Number.parseInt(namedStyleType.slice('HEADING_'.length), 10);
|
|
82
|
+
if (Number.isNaN(level) || level < 1) {
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
return '#'.repeat(Math.min(level, 6));
|
|
86
|
+
}
|
|
87
|
+
function isOrderedListLevel(level) {
|
|
88
|
+
const glyphType = level?.glyphType?.toUpperCase();
|
|
89
|
+
if (glyphType) {
|
|
90
|
+
return (glyphType.includes('NUMBER') ||
|
|
91
|
+
glyphType.includes('DECIMAL') ||
|
|
92
|
+
glyphType.includes('ALPHA') ||
|
|
93
|
+
glyphType.includes('ROMAN') ||
|
|
94
|
+
glyphType.includes('LATIN'));
|
|
95
|
+
}
|
|
96
|
+
const glyphSymbol = level?.glyphSymbol?.trim();
|
|
97
|
+
return glyphSymbol === '%0.' || glyphSymbol === '%0)' || glyphSymbol === '1.' || glyphSymbol === '1)';
|
|
98
|
+
}
|
|
99
|
+
function listPrefix(paragraph, lists) {
|
|
100
|
+
if (!paragraph.bullet) {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
const level = Math.max(paragraph.bullet.nestingLevel ?? 0, 0);
|
|
104
|
+
const indent = ' '.repeat(level);
|
|
105
|
+
const listDefinition = paragraph.bullet.listId ? lists?.[paragraph.bullet.listId] : undefined;
|
|
106
|
+
const listLevel = listDefinition?.listProperties?.nestingLevels?.[level];
|
|
107
|
+
return `${indent}${isOrderedListLevel(listLevel) ? '1. ' : '- '}`;
|
|
108
|
+
}
|
|
109
|
+
function paragraphToMarkdown(paragraph, lists) {
|
|
110
|
+
const text = paragraphText(paragraph);
|
|
111
|
+
if (!text) {
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
const prefix = paragraph ? listPrefix(paragraph, lists) : null;
|
|
115
|
+
if (prefix) {
|
|
116
|
+
return `${prefix}${text}`;
|
|
117
|
+
}
|
|
118
|
+
const heading = headingPrefix(paragraph?.paragraphStyle?.namedStyleType);
|
|
119
|
+
if (heading) {
|
|
120
|
+
const headingLine = `${heading} ${text}`;
|
|
121
|
+
const headingId = paragraph?.paragraphStyle?.headingId?.trim();
|
|
122
|
+
return headingId ? `<a id="heading-${headingId}"></a>\n${headingLine}` : headingLine;
|
|
123
|
+
}
|
|
124
|
+
return text;
|
|
125
|
+
}
|
|
126
|
+
function normalizeTableCell(cell, lists) {
|
|
127
|
+
const blocks = normalizeStructuralElements(cell?.content ?? [], lists);
|
|
128
|
+
return blocks
|
|
129
|
+
.map((block) => block.replace(/\n/g, ' <br> '))
|
|
130
|
+
.join(' / ')
|
|
131
|
+
.replace(/\|/g, '\\|')
|
|
132
|
+
.trim();
|
|
133
|
+
}
|
|
134
|
+
function markdownTableDivider(columnCount) {
|
|
135
|
+
return `| ${Array.from({ length: columnCount }, () => '---').join(' | ')} |`;
|
|
136
|
+
}
|
|
137
|
+
function normalizeTable(table, lists) {
|
|
138
|
+
const rows = table?.tableRows ?? [];
|
|
139
|
+
const normalizedRows = rows
|
|
140
|
+
.map((row) => (row.tableCells ?? []).map((cell) => normalizeTableCell(cell, lists)))
|
|
141
|
+
.filter((cells) => cells.length > 0);
|
|
142
|
+
if (normalizedRows.length === 0) {
|
|
143
|
+
return [];
|
|
144
|
+
}
|
|
145
|
+
const columnCount = Math.max(...normalizedRows.map((cells) => cells.length));
|
|
146
|
+
const paddedRows = normalizedRows.map((cells) => Array.from({ length: columnCount }, (_, index) => cells[index] ?? ''));
|
|
147
|
+
const [header, ...body] = paddedRows;
|
|
148
|
+
const blocks = [`| ${header.join(' | ')} |`, markdownTableDivider(columnCount)];
|
|
149
|
+
for (const row of body) {
|
|
150
|
+
blocks.push(`| ${row.join(' | ')} |`);
|
|
151
|
+
}
|
|
152
|
+
return [blocks.join('\n')];
|
|
153
|
+
}
|
|
154
|
+
function normalizeStructuralElements(elements, lists) {
|
|
155
|
+
const blocks = [];
|
|
156
|
+
for (const element of elements) {
|
|
157
|
+
const line = paragraphToMarkdown(element.paragraph, lists);
|
|
158
|
+
if (line) {
|
|
159
|
+
blocks.push(line);
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
if (element.table) {
|
|
163
|
+
blocks.push(...normalizeTable(element.table, lists));
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return blocks;
|
|
167
|
+
}
|
|
168
|
+
function headerFooterRoleMap(label, documentStyle) {
|
|
169
|
+
const roleMap = new Map();
|
|
170
|
+
const roleEntries = label === 'Headers'
|
|
171
|
+
? [
|
|
172
|
+
[documentStyle?.defaultHeaderId, 'Default Header'],
|
|
173
|
+
[documentStyle?.firstPageHeaderId, 'First Page Header'],
|
|
174
|
+
[documentStyle?.evenPageHeaderId, 'Even Page Header'],
|
|
175
|
+
]
|
|
176
|
+
: [
|
|
177
|
+
[documentStyle?.defaultFooterId, 'Default Footer'],
|
|
178
|
+
[documentStyle?.firstPageFooterId, 'First Page Footer'],
|
|
179
|
+
[documentStyle?.evenPageFooterId, 'Even Page Footer'],
|
|
180
|
+
];
|
|
181
|
+
for (const [id, role] of roleEntries) {
|
|
182
|
+
const normalizedId = id?.trim();
|
|
183
|
+
if (!normalizedId || roleMap.has(normalizedId)) {
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
roleMap.set(normalizedId, role ?? normalizedId);
|
|
187
|
+
}
|
|
188
|
+
return roleMap;
|
|
189
|
+
}
|
|
190
|
+
function normalizeHeaderFooterMap(label, entries, lists, documentStyle) {
|
|
191
|
+
if (!entries) {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
const ids = Object.keys(entries).sort();
|
|
195
|
+
const roles = headerFooterRoleMap(label, documentStyle);
|
|
196
|
+
const sections = [];
|
|
197
|
+
for (const id of ids) {
|
|
198
|
+
const blocks = normalizeStructuralElements(entries[id]?.content ?? [], lists);
|
|
199
|
+
if (blocks.length === 0) {
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
const title = roles.get(id) ?? `${label.slice(0, -1)} ${escapeMarkdownText(id)}`;
|
|
203
|
+
sections.push(`### ${title}\n\n${blocks.join('\n\n').trim()}`);
|
|
204
|
+
}
|
|
205
|
+
if (sections.length === 0) {
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
return `## ${label}\n\n${sections.join('\n\n').trim()}`;
|
|
209
|
+
}
|
|
210
|
+
function joinNonEmptySections(sections) {
|
|
211
|
+
const nonEmpty = sections.filter((section) => Boolean(section?.trim()));
|
|
212
|
+
if (nonEmpty.length === 0) {
|
|
213
|
+
return null;
|
|
214
|
+
}
|
|
215
|
+
return nonEmpty.join('\n\n').trim();
|
|
216
|
+
}
|
|
217
|
+
function flattenGoogleDocsTabs(tabs) {
|
|
218
|
+
if (!tabs?.length) {
|
|
219
|
+
return [];
|
|
220
|
+
}
|
|
221
|
+
const flattened = [];
|
|
222
|
+
for (const tab of tabs) {
|
|
223
|
+
flattened.push(tab);
|
|
224
|
+
flattened.push(...flattenGoogleDocsTabs(tab.childTabs));
|
|
225
|
+
}
|
|
226
|
+
return flattened;
|
|
227
|
+
}
|
|
228
|
+
function normalizeTab(tab, fallbackLists) {
|
|
229
|
+
const lists = tab.documentTab?.lists ?? fallbackLists;
|
|
230
|
+
const headerSection = normalizeHeaderFooterMap('Headers', tab.documentTab?.headers, lists, tab.documentTab?.documentStyle);
|
|
231
|
+
const bodySection = normalizeStructuralElements(tab.documentTab?.body?.content ?? [], lists).join('\n\n').trim();
|
|
232
|
+
const footerSection = normalizeHeaderFooterMap('Footers', tab.documentTab?.footers, lists, tab.documentTab?.documentStyle);
|
|
233
|
+
const content = joinNonEmptySections([headerSection, bodySection, footerSection]);
|
|
234
|
+
if (!content) {
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
const title = tab.tabProperties?.title?.trim();
|
|
238
|
+
if (!title) {
|
|
239
|
+
return content;
|
|
240
|
+
}
|
|
241
|
+
return [`# ${escapeMarkdownText(title)}`, content].join('\n\n').trim();
|
|
242
|
+
}
|
|
243
|
+
export function normalizeGoogleDocToMarkdown(document) {
|
|
244
|
+
const normalizedTabs = flattenGoogleDocsTabs(document.tabs)
|
|
245
|
+
.map((tab) => normalizeTab(tab, document.lists))
|
|
246
|
+
.filter((tab) => Boolean(tab));
|
|
247
|
+
if (normalizedTabs.length > 0) {
|
|
248
|
+
return normalizedTabs.join('\n\n').trim();
|
|
249
|
+
}
|
|
250
|
+
const bodySection = normalizeStructuralElements(document.body?.content ?? [], document.lists).join('\n\n').trim();
|
|
251
|
+
return (joinNonEmptySections([
|
|
252
|
+
normalizeHeaderFooterMap('Headers', document.headers, document.lists, document.documentStyle),
|
|
253
|
+
bodySection,
|
|
254
|
+
normalizeHeaderFooterMap('Footers', document.footers, document.lists, document.documentStyle),
|
|
255
|
+
]) ?? '');
|
|
256
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const GDRIVE_SCOPES: readonly ["https://www.googleapis.com/auth/drive.readonly", "https://www.googleapis.com/auth/documents.readonly"];
|
|
3
|
+
export declare const GDRIVE_SOURCE_KEY = "gdrive";
|
|
4
|
+
export declare const GDRIVE_DOC_MIME_TYPE = "application/vnd.google-apps.document";
|
|
5
|
+
export declare const GDRIVE_FOLDER_MIME_TYPE = "application/vnd.google-apps.folder";
|
|
6
|
+
export declare const gdrivePullConfigSchema: z.ZodObject<{
|
|
7
|
+
serviceAccountKey: z.ZodString;
|
|
8
|
+
folderId: z.ZodString;
|
|
9
|
+
recursive: z.ZodDefault<z.ZodBoolean>;
|
|
10
|
+
}, z.core.$strip>;
|
|
11
|
+
export type GdrivePullConfig = z.infer<typeof gdrivePullConfigSchema>;
|
|
12
|
+
export declare const gdriveManifestSchema: z.ZodObject<{
|
|
13
|
+
source: z.ZodLiteral<"gdrive">;
|
|
14
|
+
folderId: z.ZodString;
|
|
15
|
+
recursive: z.ZodBoolean;
|
|
16
|
+
fetchedAt: z.ZodString;
|
|
17
|
+
fileCount: z.ZodNumber;
|
|
18
|
+
skipped: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
19
|
+
externalId: z.ZodString;
|
|
20
|
+
reason: z.ZodString;
|
|
21
|
+
}, z.core.$strip>>>;
|
|
22
|
+
warnings: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
23
|
+
}, z.core.$strip>;
|
|
24
|
+
export type GdriveManifest = z.infer<typeof gdriveManifestSchema>;
|
|
25
|
+
export declare const gdriveMetadataSchema: z.ZodObject<{
|
|
26
|
+
id: z.ZodString;
|
|
27
|
+
title: z.ZodString;
|
|
28
|
+
path: z.ZodString;
|
|
29
|
+
url: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
30
|
+
mimeType: z.ZodLiteral<"application/vnd.google-apps.document">;
|
|
31
|
+
folderId: z.ZodString;
|
|
32
|
+
drivePath: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
33
|
+
modifiedTime: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
34
|
+
}, z.core.$strip>;
|
|
35
|
+
export declare const gdriveServiceAccountKeySchema: z.ZodObject<{
|
|
36
|
+
client_email: z.ZodString;
|
|
37
|
+
private_key: z.ZodString;
|
|
38
|
+
project_id: z.ZodOptional<z.ZodString>;
|
|
39
|
+
}, z.core.$strip>;
|
|
40
|
+
export type GdriveServiceAccountKey = z.infer<typeof gdriveServiceAccountKeySchema>;
|
|
41
|
+
export interface GdriveFileRecord {
|
|
42
|
+
id: string;
|
|
43
|
+
name: string;
|
|
44
|
+
mimeType: string;
|
|
45
|
+
parents: string[];
|
|
46
|
+
webViewLink: string | null;
|
|
47
|
+
modifiedTime: string | null;
|
|
48
|
+
}
|
|
49
|
+
export interface GoogleDocsDocument {
|
|
50
|
+
documentId?: string;
|
|
51
|
+
title?: string;
|
|
52
|
+
body?: {
|
|
53
|
+
content?: GoogleDocsStructuralElement[];
|
|
54
|
+
};
|
|
55
|
+
documentStyle?: GoogleDocsDocumentStyle;
|
|
56
|
+
lists?: Record<string, GoogleDocsList>;
|
|
57
|
+
headers?: Record<string, GoogleDocsHeaderFooter>;
|
|
58
|
+
footers?: Record<string, GoogleDocsHeaderFooter>;
|
|
59
|
+
tabs?: GoogleDocsTab[];
|
|
60
|
+
}
|
|
61
|
+
export interface GoogleDocsList {
|
|
62
|
+
listProperties?: {
|
|
63
|
+
nestingLevels?: GoogleDocsListNestingLevel[];
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
interface GoogleDocsListNestingLevel {
|
|
67
|
+
glyphType?: string;
|
|
68
|
+
glyphSymbol?: string;
|
|
69
|
+
}
|
|
70
|
+
export interface GoogleDocsTab {
|
|
71
|
+
tabProperties?: {
|
|
72
|
+
tabId?: string;
|
|
73
|
+
title?: string;
|
|
74
|
+
};
|
|
75
|
+
childTabs?: GoogleDocsTab[];
|
|
76
|
+
documentTab?: {
|
|
77
|
+
body?: {
|
|
78
|
+
content?: GoogleDocsStructuralElement[];
|
|
79
|
+
};
|
|
80
|
+
documentStyle?: GoogleDocsDocumentStyle;
|
|
81
|
+
lists?: Record<string, GoogleDocsList>;
|
|
82
|
+
headers?: Record<string, GoogleDocsHeaderFooter>;
|
|
83
|
+
footers?: Record<string, GoogleDocsHeaderFooter>;
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
export interface GoogleDocsDocumentStyle {
|
|
87
|
+
defaultHeaderId?: string;
|
|
88
|
+
defaultFooterId?: string;
|
|
89
|
+
firstPageHeaderId?: string;
|
|
90
|
+
firstPageFooterId?: string;
|
|
91
|
+
evenPageHeaderId?: string;
|
|
92
|
+
evenPageFooterId?: string;
|
|
93
|
+
}
|
|
94
|
+
export interface GoogleDocsHeaderFooter {
|
|
95
|
+
headerId?: string;
|
|
96
|
+
footerId?: string;
|
|
97
|
+
content?: GoogleDocsStructuralElement[];
|
|
98
|
+
}
|
|
99
|
+
export interface GoogleDocsStructuralElement {
|
|
100
|
+
paragraph?: GoogleDocsParagraph;
|
|
101
|
+
table?: GoogleDocsTable;
|
|
102
|
+
sectionBreak?: unknown;
|
|
103
|
+
}
|
|
104
|
+
export interface GoogleDocsTable {
|
|
105
|
+
tableRows?: GoogleDocsTableRow[];
|
|
106
|
+
}
|
|
107
|
+
interface GoogleDocsTableRow {
|
|
108
|
+
tableCells?: GoogleDocsTableCell[];
|
|
109
|
+
}
|
|
110
|
+
export interface GoogleDocsTableCell {
|
|
111
|
+
content?: GoogleDocsStructuralElement[];
|
|
112
|
+
}
|
|
113
|
+
export interface GoogleDocsParagraph {
|
|
114
|
+
elements?: GoogleDocsParagraphElement[];
|
|
115
|
+
bullet?: {
|
|
116
|
+
listId?: string;
|
|
117
|
+
nestingLevel?: number;
|
|
118
|
+
};
|
|
119
|
+
paragraphStyle?: {
|
|
120
|
+
namedStyleType?: string;
|
|
121
|
+
headingId?: string;
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
export interface GoogleDocsLinkTarget {
|
|
125
|
+
id?: string;
|
|
126
|
+
tabId?: string;
|
|
127
|
+
}
|
|
128
|
+
export interface GoogleDocsParagraphElement {
|
|
129
|
+
textRun?: {
|
|
130
|
+
content?: string;
|
|
131
|
+
textStyle?: {
|
|
132
|
+
bold?: boolean;
|
|
133
|
+
italic?: boolean;
|
|
134
|
+
underline?: boolean;
|
|
135
|
+
strikethrough?: boolean;
|
|
136
|
+
link?: {
|
|
137
|
+
url?: string;
|
|
138
|
+
tabId?: string;
|
|
139
|
+
headingId?: string;
|
|
140
|
+
bookmarkId?: string;
|
|
141
|
+
heading?: GoogleDocsLinkTarget;
|
|
142
|
+
bookmark?: GoogleDocsLinkTarget;
|
|
143
|
+
};
|
|
144
|
+
weightedFontFamily?: {
|
|
145
|
+
fontFamily?: string;
|
|
146
|
+
};
|
|
147
|
+
baselineOffset?: 'SUPERSCRIPT' | 'SUBSCRIPT' | string;
|
|
148
|
+
};
|
|
149
|
+
};
|
|
150
|
+
inlineObjectElement?: unknown;
|
|
151
|
+
pageBreak?: unknown;
|
|
152
|
+
}
|
|
153
|
+
export {};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
const GDRIVE_DOCS_SCOPE = 'https://www.googleapis.com/auth/documents.readonly';
|
|
3
|
+
const GDRIVE_DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.readonly';
|
|
4
|
+
export const GDRIVE_SCOPES = [GDRIVE_DRIVE_SCOPE, GDRIVE_DOCS_SCOPE];
|
|
5
|
+
export const GDRIVE_SOURCE_KEY = 'gdrive';
|
|
6
|
+
export const GDRIVE_DOC_MIME_TYPE = 'application/vnd.google-apps.document';
|
|
7
|
+
export const GDRIVE_FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder';
|
|
8
|
+
export const gdrivePullConfigSchema = z.object({
|
|
9
|
+
serviceAccountKey: z.string().min(1),
|
|
10
|
+
folderId: z.string().min(1),
|
|
11
|
+
recursive: z.boolean().default(false),
|
|
12
|
+
});
|
|
13
|
+
export const gdriveManifestSchema = z.object({
|
|
14
|
+
source: z.literal(GDRIVE_SOURCE_KEY),
|
|
15
|
+
folderId: z.string().min(1),
|
|
16
|
+
recursive: z.boolean(),
|
|
17
|
+
fetchedAt: z.string().datetime(),
|
|
18
|
+
fileCount: z.number().int().nonnegative(),
|
|
19
|
+
skipped: z.array(z.object({ externalId: z.string(), reason: z.string() })).default([]),
|
|
20
|
+
warnings: z.array(z.string()).default([]),
|
|
21
|
+
});
|
|
22
|
+
export const gdriveMetadataSchema = z.object({
|
|
23
|
+
id: z.string(),
|
|
24
|
+
title: z.string(),
|
|
25
|
+
path: z.string(),
|
|
26
|
+
url: z.string().nullable().default(null),
|
|
27
|
+
mimeType: z.literal(GDRIVE_DOC_MIME_TYPE),
|
|
28
|
+
folderId: z.string(),
|
|
29
|
+
drivePath: z.array(z.string()).default([]),
|
|
30
|
+
modifiedTime: z.string().datetime().nullable().default(null),
|
|
31
|
+
});
|
|
32
|
+
export const gdriveServiceAccountKeySchema = z.object({
|
|
33
|
+
client_email: z.string().email(),
|
|
34
|
+
private_key: z.string().min(1),
|
|
35
|
+
project_id: z.string().min(1).optional(),
|
|
36
|
+
});
|
|
@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process';
|
|
|
2
2
|
import { request as httpRequest } from 'node:http';
|
|
3
3
|
import { request as httpsRequest } from 'node:https';
|
|
4
4
|
import { URL } from 'node:url';
|
|
5
|
+
import { isKtxScanWarningCode } from '../../../scan/local-structural-artifacts.js';
|
|
5
6
|
import { tableRefFromKey } from '../../../scan/table-ref.js';
|
|
6
7
|
import { inferKtxDimensionType, normalizeKtxNativeType } from '../../../scan/type-normalization.js';
|
|
7
8
|
const DEFAULT_SCHEMAS = ['public'];
|
|
@@ -147,7 +148,29 @@ function mapTable(raw) {
|
|
|
147
148
|
foreignKeys: recordArray(raw.foreign_keys).map(mapForeignKey),
|
|
148
149
|
};
|
|
149
150
|
}
|
|
151
|
+
function mapWarning(raw) {
|
|
152
|
+
const code = optionalString(raw.code);
|
|
153
|
+
// Drop codes Node cannot render, keeping the daemon and Node warning catalogs
|
|
154
|
+
// in parity rather than surfacing an unknown code downstream.
|
|
155
|
+
if (!code || !isKtxScanWarningCode(code))
|
|
156
|
+
return null;
|
|
157
|
+
const table = optionalString(raw.table);
|
|
158
|
+
const column = optionalString(raw.column);
|
|
159
|
+
return {
|
|
160
|
+
code,
|
|
161
|
+
message: requiredString(raw.message, 'warnings[].message'),
|
|
162
|
+
recoverable: raw.recoverable !== false,
|
|
163
|
+
...(table ? { table } : {}),
|
|
164
|
+
...(column ? { column } : {}),
|
|
165
|
+
...(raw.metadata && typeof raw.metadata === 'object' && !Array.isArray(raw.metadata)
|
|
166
|
+
? { metadata: recordValue(raw.metadata) }
|
|
167
|
+
: {}),
|
|
168
|
+
};
|
|
169
|
+
}
|
|
150
170
|
function mapDaemonSnapshot(raw, input) {
|
|
171
|
+
const warnings = recordArray(raw.warnings)
|
|
172
|
+
.map(mapWarning)
|
|
173
|
+
.filter((warning) => warning !== null);
|
|
151
174
|
return {
|
|
152
175
|
connectionId: requiredString(raw.connection_id, 'connection_id') || input.connectionId,
|
|
153
176
|
driver: 'postgres',
|
|
@@ -155,6 +178,7 @@ function mapDaemonSnapshot(raw, input) {
|
|
|
155
178
|
scope: { schemas: input.schemas },
|
|
156
179
|
metadata: recordValue(raw.metadata),
|
|
157
180
|
tables: recordArray(raw.tables).map(mapTable),
|
|
181
|
+
...(warnings.length > 0 ? { warnings } : {}),
|
|
158
182
|
};
|
|
159
183
|
}
|
|
160
184
|
function serializeTableScope(options) {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { SourceFetchReport } from '../../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Derives the fetch report from the staged `warnings.json`: objects that failed
|
|
4
|
+
* introspection become `skipped` entries so the run report, ingest summary, and
|
|
5
|
+
* `ktx status` can surface them. Returns null when nothing was skipped, keeping
|
|
6
|
+
* clean ingests free of an empty report.
|
|
7
|
+
*/
|
|
8
|
+
export declare function readLiveDatabaseFetchReport(stagedDir: string): Promise<SourceFetchReport | null>;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { LIVE_DATABASE_WARNINGS_FILE } from './stage.js';
|
|
4
|
+
const OBJECT_SKIP_CODE = 'object_introspection_failed';
|
|
5
|
+
/**
|
|
6
|
+
* Derives the fetch report from the staged `warnings.json`: objects that failed
|
|
7
|
+
* introspection become `skipped` entries so the run report, ingest summary, and
|
|
8
|
+
* `ktx status` can surface them. Returns null when nothing was skipped, keeping
|
|
9
|
+
* clean ingests free of an empty report.
|
|
10
|
+
*/
|
|
11
|
+
export async function readLiveDatabaseFetchReport(stagedDir) {
|
|
12
|
+
let parsed;
|
|
13
|
+
try {
|
|
14
|
+
parsed = JSON.parse(await readFile(join(stagedDir, LIVE_DATABASE_WARNINGS_FILE), 'utf8'));
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
const warnings = parsed && typeof parsed === 'object' && Array.isArray(parsed.warnings)
|
|
20
|
+
? (parsed.warnings)
|
|
21
|
+
: [];
|
|
22
|
+
const skipped = warnings
|
|
23
|
+
.filter((warning) => warning.code === OBJECT_SKIP_CODE)
|
|
24
|
+
.map((warning) => ({
|
|
25
|
+
rawPath: '',
|
|
26
|
+
entityType: 'database_object',
|
|
27
|
+
entityId: typeof warning.table === 'string' ? warning.table : null,
|
|
28
|
+
severity: 'warning',
|
|
29
|
+
statusCode: null,
|
|
30
|
+
message: typeof warning.message === 'string' ? warning.message : 'introspection failed',
|
|
31
|
+
retryRecommended: false,
|
|
32
|
+
}));
|
|
33
|
+
if (skipped.length === 0) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
return { status: 'partial', retryRecommended: false, skipped, warnings: [] };
|
|
37
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ChunkResult, DiffSet, FetchContext, SourceAdapter } from '../../types.js';
|
|
1
|
+
import type { ChunkResult, DiffSet, FetchContext, SourceAdapter, SourceFetchReport } from '../../types.js';
|
|
2
2
|
import type { LiveDatabaseSourceAdapterDeps } from './types.js';
|
|
3
3
|
export declare class LiveDatabaseSourceAdapter implements SourceAdapter {
|
|
4
4
|
private readonly deps;
|
|
@@ -6,6 +6,7 @@ export declare class LiveDatabaseSourceAdapter implements SourceAdapter {
|
|
|
6
6
|
readonly skillNames: string[];
|
|
7
7
|
constructor(deps: LiveDatabaseSourceAdapterDeps);
|
|
8
8
|
detect(stagedDir: string): Promise<boolean>;
|
|
9
|
+
readFetchReport(stagedDir: string): Promise<SourceFetchReport | null>;
|
|
9
10
|
fetch(_pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise<void>;
|
|
10
11
|
chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
|
|
11
12
|
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { chunkLiveDatabaseStagedDir } from './chunk.js';
|
|
2
|
+
import { readLiveDatabaseFetchReport } from './fetch-report.js';
|
|
3
|
+
import { assertLiveDatabaseScanOutcome } from './scan-outcome.js';
|
|
2
4
|
import { detectLiveDatabaseStagedDir, writeLiveDatabaseSnapshot } from './stage.js';
|
|
3
5
|
export class LiveDatabaseSourceAdapter {
|
|
4
6
|
deps;
|
|
@@ -10,14 +12,19 @@ export class LiveDatabaseSourceAdapter {
|
|
|
10
12
|
detect(stagedDir) {
|
|
11
13
|
return detectLiveDatabaseStagedDir(stagedDir);
|
|
12
14
|
}
|
|
15
|
+
readFetchReport(stagedDir) {
|
|
16
|
+
return readLiveDatabaseFetchReport(stagedDir);
|
|
17
|
+
}
|
|
13
18
|
async fetch(_pullConfig, stagedDir, ctx) {
|
|
14
19
|
const tableScope = ctx.tableScope;
|
|
15
20
|
const snapshot = await this.deps.introspection.extractSchema(ctx.connectionId, { tableScope });
|
|
16
|
-
|
|
21
|
+
const finalized = {
|
|
17
22
|
...snapshot,
|
|
18
23
|
connectionId: ctx.connectionId,
|
|
19
24
|
extractedAt: snapshot.extractedAt ?? (this.deps.now ?? (() => new Date()))().toISOString(),
|
|
20
|
-
}
|
|
25
|
+
};
|
|
26
|
+
assertLiveDatabaseScanOutcome({ connectionId: ctx.connectionId, scope: tableScope, snapshot: finalized });
|
|
27
|
+
await writeLiveDatabaseSnapshot(stagedDir, finalized);
|
|
21
28
|
}
|
|
22
29
|
chunk(stagedDir, diffSet) {
|
|
23
30
|
return chunkLiveDatabaseStagedDir(stagedDir, diffSet);
|
|
@@ -64,6 +64,8 @@ export interface BuildLiveDatabaseManifestShardsResult {
|
|
|
64
64
|
}
|
|
65
65
|
export declare function mergeUsagePreservingExternal(existing: TableUsageOutput | undefined, incoming: TableUsageOutput | undefined): TableUsageOutput | undefined;
|
|
66
66
|
/** @internal */
|
|
67
|
+
export declare function buildTableRef(name: string, catalog: string | null, db: string | null): string;
|
|
68
|
+
/** @internal */
|
|
67
69
|
export declare function buildJoinsByTable(tableNames: Set<string>, joins: LiveDatabaseManifestJoinData[], preservedJoins: Map<string, LiveDatabaseManifestJoinEntry[]>, federatedSiblingTargets?: Set<string>): Map<string, LiveDatabaseManifestJoinEntry[]>;
|
|
68
70
|
export declare function buildLiveDatabaseManifestShards(input: BuildLiveDatabaseManifestShardsInput): BuildLiveDatabaseManifestShardsResult;
|
|
69
71
|
export {};
|
|
@@ -73,7 +73,8 @@ function getShardKey(connectionType, catalog, db) {
|
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
|
-
|
|
76
|
+
/** @internal */
|
|
77
|
+
export function buildTableRef(name, catalog, db) {
|
|
77
78
|
const parts = [];
|
|
78
79
|
if (catalog) {
|
|
79
80
|
parts.push(catalog);
|
|
@@ -154,7 +155,10 @@ export function buildLiveDatabaseManifestShards(input) {
|
|
|
154
155
|
for (const table of input.tables) {
|
|
155
156
|
const shardKey = getShardKey(input.connectionType, table.catalog, table.db);
|
|
156
157
|
const shard = shards.get(shardKey) ?? { tables: {} };
|
|
157
|
-
|
|
158
|
+
// Existing descriptions/usage are keyed by the fully-qualified ref so two
|
|
159
|
+
// same-named tables in different schemas never share an entry.
|
|
160
|
+
const fullRef = buildTableRef(table.name, table.catalog, table.db);
|
|
161
|
+
const existingDescriptions = input.existingDescriptions?.get(fullRef);
|
|
158
162
|
const columns = table.columns.map((column) => {
|
|
159
163
|
const manifestColumn = {
|
|
160
164
|
name: column.name,
|
|
@@ -173,14 +177,14 @@ export function buildLiveDatabaseManifestShards(input) {
|
|
|
173
177
|
return manifestColumn;
|
|
174
178
|
});
|
|
175
179
|
const entry = {
|
|
176
|
-
table:
|
|
180
|
+
table: fullRef,
|
|
177
181
|
columns,
|
|
178
182
|
};
|
|
179
183
|
const tableDescriptions = mergeDescriptionsPreservingExternal(existingDescriptions?.table, table.descriptions);
|
|
180
184
|
if (tableDescriptions) {
|
|
181
185
|
entry.descriptions = tableDescriptions;
|
|
182
186
|
}
|
|
183
|
-
const usage = mergeUsagePreservingExternal(input.existingUsage?.get(
|
|
187
|
+
const usage = mergeUsagePreservingExternal(input.existingUsage?.get(fullRef), table.usage);
|
|
184
188
|
if (usage) {
|
|
185
189
|
entry.usage = usage;
|
|
186
190
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type KtxTableRefKey } from '../../../scan/table-ref.js';
|
|
2
|
+
import type { KtxSchemaSnapshot } from '../../../scan/types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Enforces the partial-vs-total outcome rules for a live-database snapshot,
|
|
5
|
+
* uniformly for every connector. Outcomes follow from object counts, not a
|
|
6
|
+
* mode: a connection with at least one ingested object succeeds (any broken
|
|
7
|
+
* objects ride along as warnings); a connection where every introspected object
|
|
8
|
+
* failed, or a non-empty enabled_tables scope that matched nothing, raises a
|
|
9
|
+
* clear connection error instead of staging an empty layer that would later
|
|
10
|
+
* surface as the generic "did not recognize" message. A legitimately empty
|
|
11
|
+
* database (no scope, no objects) succeeds with an empty layer.
|
|
12
|
+
*/
|
|
13
|
+
export declare function assertLiveDatabaseScanOutcome(input: {
|
|
14
|
+
connectionId: string;
|
|
15
|
+
scope: ReadonlySet<KtxTableRefKey> | undefined;
|
|
16
|
+
snapshot: KtxSchemaSnapshot;
|
|
17
|
+
}): void;
|