@kaelio/ktx 0.13.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.13.1-py3-none-any.whl → kaelio_ktx-0.15.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +1 -1
- package/dist/commands/ingest-commands.d.ts +9 -0
- package/dist/commands/ingest-commands.js +37 -1
- package/dist/commands/knowledge-commands.js +3 -0
- package/dist/commands/setup-commands.js +15 -1
- package/dist/connection-drivers.d.ts +2 -0
- package/dist/connection-drivers.js +7 -3
- package/dist/connection.d.ts +3 -0
- package/dist/connection.js +27 -0
- package/dist/connectors/bigquery/connector.d.ts +17 -6
- package/dist/connectors/bigquery/connector.js +152 -76
- package/dist/connectors/bigquery/dialect.d.ts +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +1 -0
- package/dist/connectors/clickhouse/connector.js +45 -16
- package/dist/connectors/clickhouse/dialect.d.ts +2 -2
- package/dist/connectors/mongodb/connector.d.ts +69 -0
- package/dist/connectors/mongodb/connector.js +307 -0
- package/dist/connectors/mongodb/dialect.d.ts +17 -0
- package/dist/connectors/mongodb/dialect.js +50 -0
- package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
- package/dist/connectors/mongodb/live-database-introspection.js +24 -0
- package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
- package/dist/connectors/mongodb/schema-inference.js +110 -0
- package/dist/connectors/mysql/connector.d.ts +1 -0
- package/dist/connectors/mysql/connector.js +18 -2
- package/dist/connectors/mysql/dialect.d.ts +2 -2
- package/dist/connectors/postgres/connector.d.ts +1 -0
- package/dist/connectors/postgres/connector.js +20 -3
- package/dist/connectors/postgres/dialect.d.ts +2 -2
- package/dist/connectors/snowflake/connector.d.ts +1 -0
- package/dist/connectors/snowflake/connector.js +27 -3
- package/dist/connectors/snowflake/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/connector.d.ts +11 -1
- package/dist/connectors/sqlite/connector.js +120 -17
- package/dist/connectors/sqlite/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
- package/dist/connectors/sqlite/read-query-child.js +23 -0
- package/dist/connectors/sqlserver/connector.d.ts +2 -0
- package/dist/connectors/sqlserver/connector.js +20 -3
- package/dist/connectors/sqlserver/dialect.d.ts +2 -2
- package/dist/context/cache/content-result-cache.d.ts +36 -0
- package/dist/context/cache/content-result-cache.js +14 -0
- package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
- package/dist/context/cache/sqlite-content-result-cache.js +223 -0
- package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
- package/dist/context/connections/bigquery-identifiers.js +7 -0
- package/dist/context/connections/configured-connections.d.ts +9 -0
- package/dist/context/connections/configured-connections.js +18 -0
- package/dist/context/connections/dialects.d.ts +30 -4
- package/dist/context/connections/dialects.js +38 -11
- package/dist/context/connections/drivers.js +21 -0
- package/dist/context/connections/gdrive-config.d.ts +19 -0
- package/dist/context/connections/gdrive-config.js +57 -0
- package/dist/context/connections/query-deadline.d.ts +31 -0
- package/dist/context/connections/query-deadline.js +37 -0
- package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
- package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
- package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
- package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
- package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
- package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
- package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
- package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
- package/dist/context/ingest/adapters/gdrive/types.js +36 -0
- package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
- package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
- package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
- package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
- package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
- package/dist/context/ingest/adapters/live-database/stage.js +5 -5
- package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/types.js +1 -1
- package/dist/context/ingest/adapters/sigma/chunk.d.ts +6 -0
- package/dist/context/ingest/adapters/sigma/chunk.js +119 -0
- package/dist/context/ingest/adapters/sigma/client-port.d.ts +45 -0
- package/dist/context/ingest/adapters/sigma/client-port.js +1 -0
- package/dist/context/ingest/adapters/sigma/client.d.ts +33 -0
- package/dist/context/ingest/adapters/sigma/client.js +176 -0
- package/dist/context/ingest/adapters/sigma/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/sigma/detect.js +23 -0
- package/dist/context/ingest/adapters/sigma/fetch.d.ts +14 -0
- package/dist/context/ingest/adapters/sigma/fetch.js +191 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.d.ts +17 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.js +41 -0
- package/dist/context/ingest/adapters/sigma/project.d.ts +8 -0
- package/dist/context/ingest/adapters/sigma/project.js +186 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.d.ts +18 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.js +43 -0
- package/dist/context/ingest/adapters/sigma/types.d.ts +80 -0
- package/dist/context/ingest/adapters/sigma/types.js +82 -0
- package/dist/context/ingest/artifact-gates.d.ts +32 -1
- package/dist/context/ingest/artifact-gates.js +85 -18
- package/dist/context/ingest/final-gate-prune.d.ts +35 -0
- package/dist/context/ingest/final-gate-prune.js +229 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +459 -240
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
- package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
- package/dist/context/ingest/local-adapters.d.ts +2 -1
- package/dist/context/ingest/local-adapters.js +28 -0
- package/dist/context/ingest/local-bundle-runtime.js +7 -2
- package/dist/context/ingest/local-ingest.js +1 -1
- package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
- package/dist/context/ingest/local-stage-ingest.js +3 -1
- package/dist/context/ingest/ports.d.ts +3 -0
- package/dist/context/ingest/report-snapshot.js +13 -3
- package/dist/context/ingest/reports.d.ts +3 -3
- package/dist/context/ingest/reports.js +3 -1
- package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
- package/dist/context/ingest/stages/build-wu-context.js +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
- package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
- package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
- package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
- package/dist/context/ingest/types.d.ts +3 -0
- package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
- package/dist/context/ingest/wiki-body-refs.js +37 -8
- package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
- package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
- package/dist/context/ingest/work-unit-cache.d.ts +67 -0
- package/dist/context/ingest/work-unit-cache.js +230 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.js +5 -0
- package/dist/context/llm/claude-code-runtime.d.ts +4 -1
- package/dist/context/llm/claude-code-runtime.js +38 -8
- package/dist/context/llm/codex-runtime.d.ts +4 -1
- package/dist/context/llm/codex-runtime.js +48 -35
- package/dist/context/llm/runtime-port.d.ts +26 -0
- package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
- package/dist/context/llm/subprocess-generate-object-child.js +34 -0
- package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
- package/dist/context/llm/subprocess-generate-object.js +122 -0
- package/dist/context/mcp/context-tools.d.ts +2 -0
- package/dist/context/mcp/context-tools.js +111 -17
- package/dist/context/mcp/local-project-ports.d.ts +7 -0
- package/dist/context/mcp/local-project-ports.js +29 -0
- package/dist/context/mcp/logger.d.ts +24 -0
- package/dist/context/mcp/logger.js +49 -0
- package/dist/context/mcp/server.js +2 -0
- package/dist/context/mcp/types.d.ts +17 -0
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +72 -0
- package/dist/context/project/config.js +5 -0
- package/dist/context/project/driver-schemas.d.ts +35 -0
- package/dist/context/project/driver-schemas.js +89 -1
- package/dist/context/project/project.js +1 -1
- package/dist/context/project/setup-config.js +1 -0
- package/dist/context/scan/description-generation.d.ts +4 -0
- package/dist/context/scan/description-generation.js +100 -13
- package/dist/context/scan/enabled-tables.d.ts +5 -0
- package/dist/context/scan/enabled-tables.js +13 -1
- package/dist/context/scan/enrichment-state.d.ts +50 -7
- package/dist/context/scan/enrichment-state.js +30 -13
- package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
- package/dist/context/scan/local-enrichment-artifacts.js +155 -32
- package/dist/context/scan/local-enrichment.d.ts +39 -4
- package/dist/context/scan/local-enrichment.js +345 -92
- package/dist/context/scan/local-scan.d.ts +4 -1
- package/dist/context/scan/local-scan.js +54 -13
- package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
- package/dist/context/scan/local-structural-artifacts.js +5 -0
- package/dist/context/scan/object-introspection.d.ts +21 -0
- package/dist/context/scan/object-introspection.js +35 -0
- package/dist/context/scan/relationship-benchmarks.js +5 -3
- package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
- package/dist/context/scan/relationship-composite-candidates.js +13 -1
- package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
- package/dist/context/scan/relationship-detection-budget.js +53 -0
- package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
- package/dist/context/scan/relationship-diagnostics.js +2 -0
- package/dist/context/scan/relationship-discovery.d.ts +9 -3
- package/dist/context/scan/relationship-discovery.js +27 -4
- package/dist/context/scan/relationship-llm-proposal.js +36 -27
- package/dist/context/scan/relationship-profiling.d.ts +7 -3
- package/dist/context/scan/relationship-profiling.js +53 -41
- package/dist/context/scan/relationship-validation.d.ts +6 -4
- package/dist/context/scan/relationship-validation.js +46 -32
- package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
- package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
- package/dist/context/scan/types.d.ts +5 -3
- package/dist/context/scan/types.js +2 -0
- package/dist/context/sl/local-query.js +5 -0
- package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
- package/dist/context/sl/semantic-layer.service.js +1 -1
- package/dist/context/sl/source-files.d.ts +5 -0
- package/dist/context/sl/source-files.js +17 -11
- package/dist/context/sl/tools/connection-id-schema.js +1 -1
- package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
- package/dist/context/sql-analysis/dialect-notes.js +40 -0
- package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
- package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
- package/dist/context/sql-analysis/dialects/mysql.md +9 -0
- package/dist/context/sql-analysis/dialects/postgres.md +10 -0
- package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
- package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
- package/dist/context/sql-analysis/dialects/tsql.md +10 -0
- package/dist/context/wiki/keys.js +1 -1
- package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
- package/dist/context/wiki/knowledge-wiki.service.js +2 -1
- package/dist/context/wiki/local-knowledge.d.ts +13 -0
- package/dist/context/wiki/local-knowledge.js +50 -6
- package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
- package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
- package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
- package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
- package/dist/context/wiki/types.d.ts +6 -0
- package/dist/context-build-view.d.ts +3 -0
- package/dist/context-build-view.js +1 -0
- package/dist/knowledge.d.ts +2 -0
- package/dist/knowledge.js +12 -1
- package/dist/local-adapters.js +11 -0
- package/dist/mcp-http-server.js +15 -1
- package/dist/mcp-server-factory.d.ts +2 -0
- package/dist/mcp-server-factory.js +15 -1
- package/dist/mcp-stdio-server.js +10 -2
- package/dist/notion-page-picker.js +1 -1
- package/dist/prompts/memory_agent_external_ingest.md +2 -0
- package/dist/public-ingest.d.ts +3 -1
- package/dist/public-ingest.js +4 -0
- package/dist/scan.d.ts +3 -1
- package/dist/scan.js +7 -0
- package/dist/setup-databases.d.ts +1 -1
- package/dist/setup-databases.js +43 -1
- package/dist/setup-sources.d.ts +6 -1
- package/dist/setup-sources.js +208 -48
- package/dist/setup.d.ts +3 -0
- package/dist/setup.js +6 -1
- package/dist/skills/analytics/SKILL.md +200 -2
- package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
- package/dist/skills/sigma_ingest/SKILL.md +189 -0
- package/dist/skills/wiki_capture/SKILL.md +24 -0
- package/dist/sql.js +4 -0
- package/dist/status-project.d.ts +4 -0
- package/dist/status-project.js +54 -2
- package/dist/telemetry/events.d.ts +2 -2
- package/dist/text-ingest.d.ts +4 -0
- package/dist/text-ingest.js +58 -29
- package/dist/verbatim-ingest.d.ts +46 -0
- package/dist/verbatim-ingest.js +193 -0
- package/package.json +5 -1
- package/dist/context/ingest/final-gate-repair.d.ts +0 -28
- package/dist/context/ingest/final-gate-repair.js +0 -91
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { BigQuery } from '@google-cloud/bigquery';
|
|
2
|
-
import { normalizeBigQueryProjectId, normalizeBigQueryRegion } from '../../context/connections/bigquery-identifiers.js';
|
|
3
|
-
import {
|
|
2
|
+
import { normalizeBigQueryDatasetId, normalizeBigQueryProjectId, normalizeBigQueryRegion, } from '../../context/connections/bigquery-identifiers.js';
|
|
3
|
+
import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
|
|
4
|
+
import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
|
|
4
5
|
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
|
5
6
|
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
|
7
|
+
import { tryIntrospectObject } from '../../context/scan/object-introspection.js';
|
|
6
8
|
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
|
7
9
|
import { connectorTestFailure, createKtxConnectorCapabilities, } from '../../context/scan/types.js';
|
|
8
10
|
import { resolveStringReference } from '../shared/string-reference.js';
|
|
@@ -11,8 +13,8 @@ class DefaultBigQueryClientFactory {
|
|
|
11
13
|
const client = new BigQuery(input);
|
|
12
14
|
return {
|
|
13
15
|
getDatasets: (options) => client.getDatasets(options),
|
|
14
|
-
dataset: (datasetId) => {
|
|
15
|
-
const dataset = client.dataset(datasetId);
|
|
16
|
+
dataset: (datasetId, projectId) => {
|
|
17
|
+
const dataset = client.dataset(datasetId, { projectId });
|
|
16
18
|
return {
|
|
17
19
|
get: () => dataset.get(),
|
|
18
20
|
getTables: () => dataset.getTables(),
|
|
@@ -26,14 +28,39 @@ function stringConfigValue(connection, key, env) {
|
|
|
26
28
|
const value = connection?.[key];
|
|
27
29
|
return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
|
|
28
30
|
}
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
31
|
+
/**
|
|
32
|
+
* Parse one `dataset_ids` / `dataset_id` entry into a canonical
|
|
33
|
+
* {@link BigQueryDatasetRef}. A `project.dataset` prefix selects the host
|
|
34
|
+
* project; a bare entry defaults to `defaultProject` (the billing project).
|
|
35
|
+
* More than one dot, or an empty segment, is a config error naming the
|
|
36
|
+
* connection — never a silent mis-introspection at scan time.
|
|
37
|
+
*/
|
|
38
|
+
function parseBigQueryDatasetEntry(entry, defaultProject, connectionId) {
|
|
39
|
+
const context = `connections.${connectionId}.dataset_ids entry "${entry}"`;
|
|
40
|
+
const parts = entry.split('.');
|
|
41
|
+
if (parts.length === 1) {
|
|
42
|
+
return { project: defaultProject, dataset: normalizeBigQueryDatasetId(parts[0], context) };
|
|
43
|
+
}
|
|
44
|
+
if (parts.length === 2) {
|
|
45
|
+
const [project, dataset] = parts;
|
|
46
|
+
if (!project || !dataset) {
|
|
47
|
+
throw new Error(`Invalid BigQuery dataset entry for ${context}: empty project or dataset segment`);
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
project: normalizeBigQueryProjectId(project, context),
|
|
51
|
+
dataset: normalizeBigQueryDatasetId(dataset, context),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
throw new Error(`Invalid BigQuery dataset entry for ${context}: expected "dataset" or "project.dataset", got more than one "."`);
|
|
55
|
+
}
|
|
56
|
+
function resolveDatasetRefs(connection, env, defaultProject, connectionId) {
|
|
57
|
+
const rawEntries = Array.isArray(connection.dataset_ids) && connection.dataset_ids.length > 0
|
|
58
|
+
? connection.dataset_ids.map((dataset) => resolveStringReference(dataset, env))
|
|
59
|
+
: [stringConfigValue(connection, 'dataset_id', env)].filter((value) => Boolean(value));
|
|
60
|
+
return rawEntries
|
|
61
|
+
.map((entry) => entry.trim())
|
|
62
|
+
.filter((entry) => entry.length > 0)
|
|
63
|
+
.map((entry) => parseBigQueryDatasetEntry(entry, defaultProject, connectionId));
|
|
37
64
|
}
|
|
38
65
|
function bigQueryMaxBytesBilledFromConnection(connection) {
|
|
39
66
|
const value = connection?.max_bytes_billed;
|
|
@@ -46,12 +73,23 @@ function bigQueryMaxBytesBilledFromConnection(connection) {
|
|
|
46
73
|
}
|
|
47
74
|
return undefined;
|
|
48
75
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
76
|
+
// jobTimeoutMs cancels the job with a "Job timed out" message (or a timeout
|
|
77
|
+
// reason in the errors array) once the deadline elapses.
|
|
78
|
+
function isBigQueryTimeoutError(error) {
|
|
79
|
+
if (!error || typeof error !== 'object') {
|
|
80
|
+
return false;
|
|
53
81
|
}
|
|
54
|
-
|
|
82
|
+
const topMessage = error.message;
|
|
83
|
+
if (typeof topMessage === 'string' && /timed out|timeout/i.test(topMessage)) {
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
const errors = error.errors;
|
|
87
|
+
return (Array.isArray(errors) &&
|
|
88
|
+
errors.some((entry) => {
|
|
89
|
+
const reason = entry?.reason;
|
|
90
|
+
const message = entry?.message;
|
|
91
|
+
return reason === 'timeout' || (typeof message === 'string' && /timed out|timeout/i.test(message));
|
|
92
|
+
}));
|
|
55
93
|
}
|
|
56
94
|
function tableKind(metadataType) {
|
|
57
95
|
const type = String(metadataType ?? '').toUpperCase();
|
|
@@ -125,7 +163,7 @@ export function bigQueryConnectionConfigFromConfig(input) {
|
|
|
125
163
|
if (!projectId) {
|
|
126
164
|
throw new Error(`Native BigQuery connector requires credentials_json.project_id for connections.${input.connectionId}`);
|
|
127
165
|
}
|
|
128
|
-
const resolvedDatasetIds =
|
|
166
|
+
const resolvedDatasetIds = resolveDatasetRefs(input.connection, env, projectId, input.connectionId);
|
|
129
167
|
const location = stringConfigValue(input.connection, 'location', env);
|
|
130
168
|
return { projectId, credentials, datasetIds: resolvedDatasetIds, ...(location ? { location } : {}) };
|
|
131
169
|
}
|
|
@@ -146,8 +184,8 @@ export class KtxBigQueryScanConnector {
|
|
|
146
184
|
clientFactory;
|
|
147
185
|
now;
|
|
148
186
|
maxBytesBilled;
|
|
149
|
-
|
|
150
|
-
dialect =
|
|
187
|
+
deadlineMs;
|
|
188
|
+
dialect = getSqlDialectForDriver('bigquery');
|
|
151
189
|
client = null;
|
|
152
190
|
constructor(options) {
|
|
153
191
|
this.connectionId = options.connectionId;
|
|
@@ -159,15 +197,15 @@ export class KtxBigQueryScanConnector {
|
|
|
159
197
|
this.clientFactory = options.clientFactory ?? new DefaultBigQueryClientFactory();
|
|
160
198
|
this.now = options.now ?? (() => new Date());
|
|
161
199
|
this.maxBytesBilled = options.maxBytesBilled ?? bigQueryMaxBytesBilledFromConnection(options.connection);
|
|
162
|
-
this.
|
|
200
|
+
this.deadlineMs = resolveQueryDeadlineMs(options.connection);
|
|
163
201
|
this.id = `bigquery:${options.connectionId}`;
|
|
164
202
|
}
|
|
165
203
|
async testConnection() {
|
|
166
204
|
try {
|
|
167
205
|
const client = this.getClient();
|
|
168
206
|
await client.getDatasets({ maxResults: 1 });
|
|
169
|
-
for (const
|
|
170
|
-
await client.dataset(
|
|
207
|
+
for (const ref of this.resolved.datasetIds) {
|
|
208
|
+
await client.dataset(ref.dataset, ref.project).get();
|
|
171
209
|
}
|
|
172
210
|
return { success: true };
|
|
173
211
|
}
|
|
@@ -178,22 +216,23 @@ export class KtxBigQueryScanConnector {
|
|
|
178
216
|
async introspect(input, _ctx) {
|
|
179
217
|
this.assertConnection(input.connectionId);
|
|
180
218
|
const tables = [];
|
|
181
|
-
const
|
|
219
|
+
const datasetRefs = this.requireDatasetIdsForScan();
|
|
182
220
|
const snapshotWarnings = [];
|
|
183
|
-
for (const
|
|
221
|
+
for (const ref of datasetRefs) {
|
|
184
222
|
const scopedNames = input.tableScope
|
|
185
|
-
? scopedTableNames(input.tableScope, { catalog:
|
|
223
|
+
? scopedTableNames(input.tableScope, { catalog: ref.project, db: ref.dataset })
|
|
186
224
|
: null;
|
|
187
|
-
tables.push(...(await this.introspectDataset(
|
|
225
|
+
tables.push(...(await this.introspectDataset(ref, scopedNames, snapshotWarnings)));
|
|
188
226
|
}
|
|
227
|
+
const datasetLabels = datasetRefs.map((ref) => this.qualifiedDatasetLabel(ref));
|
|
189
228
|
return {
|
|
190
229
|
connectionId: this.connectionId,
|
|
191
230
|
driver: 'bigquery',
|
|
192
231
|
extractedAt: this.now().toISOString(),
|
|
193
|
-
scope: { catalogs: [
|
|
232
|
+
scope: { catalogs: [...new Set(datasetRefs.map((ref) => ref.project))], datasets: datasetLabels },
|
|
194
233
|
metadata: {
|
|
195
234
|
project_id: this.resolved.projectId,
|
|
196
|
-
datasets:
|
|
235
|
+
datasets: datasetLabels,
|
|
197
236
|
table_count: tables.length,
|
|
198
237
|
total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
|
|
199
238
|
},
|
|
@@ -237,11 +276,11 @@ export class KtxBigQueryScanConnector {
|
|
|
237
276
|
const valueRows = await this.queryRaw(this.dialect.generateDistinctValuesQuery(tableName, quotedColumn, options.limit));
|
|
238
277
|
return { values: valueRows.filter((row) => row.val !== null).map((row) => String(row.val)), cardinality };
|
|
239
278
|
}
|
|
240
|
-
async getTableRowCount(tableName,
|
|
241
|
-
if (!
|
|
279
|
+
async getTableRowCount(tableName, ref = this.resolved.datasetIds[0]) {
|
|
280
|
+
if (!ref) {
|
|
242
281
|
return 0;
|
|
243
282
|
}
|
|
244
|
-
const tables = await this.introspectDataset(
|
|
283
|
+
const tables = await this.introspectDataset(ref, null, []);
|
|
245
284
|
return tables.find((table) => table.name === tableName)?.estimatedRows ?? 0;
|
|
246
285
|
}
|
|
247
286
|
qTableName(table) {
|
|
@@ -255,12 +294,27 @@ export class KtxBigQueryScanConnector {
|
|
|
255
294
|
return datasets.map((dataset) => dataset.id).filter((id) => Boolean(id));
|
|
256
295
|
}
|
|
257
296
|
async listTables(datasetIds) {
|
|
258
|
-
const projectId = normalizeBigQueryProjectId(this.resolved.projectId, 'table discovery');
|
|
259
297
|
const region = normalizeBigQueryRegion(this.resolved.location ?? 'US', 'table discovery');
|
|
298
|
+
if (!datasetIds || datasetIds.length === 0) {
|
|
299
|
+
return this.listTablesInProject(this.resolved.projectId, region);
|
|
300
|
+
}
|
|
301
|
+
const datasetsByProject = new Map();
|
|
302
|
+
for (const entry of datasetIds) {
|
|
303
|
+
const ref = parseBigQueryDatasetEntry(entry.trim(), this.resolved.projectId, this.connectionId);
|
|
304
|
+
datasetsByProject.set(ref.project, [...(datasetsByProject.get(ref.project) ?? []), ref.dataset]);
|
|
305
|
+
}
|
|
306
|
+
const entries = [];
|
|
307
|
+
for (const [project, datasets] of datasetsByProject) {
|
|
308
|
+
entries.push(...(await this.listTablesInProject(project, region, datasets)));
|
|
309
|
+
}
|
|
310
|
+
return entries;
|
|
311
|
+
}
|
|
312
|
+
async listTablesInProject(project, region, datasets) {
|
|
313
|
+
const projectId = normalizeBigQueryProjectId(project, 'table discovery');
|
|
260
314
|
const params = {};
|
|
261
|
-
const filter =
|
|
262
|
-
if (
|
|
263
|
-
params.dataset_ids =
|
|
315
|
+
const filter = datasets && datasets.length > 0 ? 'AND table_schema IN UNNEST(@dataset_ids)' : '';
|
|
316
|
+
if (datasets && datasets.length > 0) {
|
|
317
|
+
params.dataset_ids = datasets;
|
|
264
318
|
}
|
|
265
319
|
const rows = await this.queryRaw(`
|
|
266
320
|
SELECT table_schema, table_name, table_type
|
|
@@ -272,7 +326,7 @@ export class KtxBigQueryScanConnector {
|
|
|
272
326
|
ORDER BY table_schema, table_name
|
|
273
327
|
`, params);
|
|
274
328
|
return rows.map((row) => ({
|
|
275
|
-
catalog:
|
|
329
|
+
catalog: project,
|
|
276
330
|
schema: row.table_schema,
|
|
277
331
|
name: row.table_name,
|
|
278
332
|
kind: row.table_type === 'VIEW' || row.table_type === 'MATERIALIZED VIEW'
|
|
@@ -298,27 +352,41 @@ export class KtxBigQueryScanConnector {
|
|
|
298
352
|
}
|
|
299
353
|
return this.resolved.datasetIds;
|
|
300
354
|
}
|
|
355
|
+
// Bare in the billing project, qualified `project.dataset` otherwise, so the
|
|
356
|
+
// snapshot's scope/metadata stay unambiguous when two projects host the same
|
|
357
|
+
// dataset name. The dotless form is the unchanged single-project label.
|
|
358
|
+
qualifiedDatasetLabel(ref) {
|
|
359
|
+
return ref.project === this.resolved.projectId ? ref.dataset : `${ref.project}.${ref.dataset}`;
|
|
360
|
+
}
|
|
301
361
|
async query(sql, params) {
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
headers
|
|
362
|
+
try {
|
|
363
|
+
const [job] = await this.getClient().createQueryJob({
|
|
364
|
+
query: sql,
|
|
365
|
+
...(this.resolved.location ? { location: this.resolved.location } : {}),
|
|
366
|
+
...(params && Object.keys(params).length > 0 ? { params } : {}),
|
|
367
|
+
...(this.maxBytesBilled ? { maximumBytesBilled: String(this.maxBytesBilled) } : {}),
|
|
368
|
+
jobTimeoutMs: this.deadlineMs,
|
|
369
|
+
});
|
|
370
|
+
const [rows, , response] = await job.getQueryResults();
|
|
371
|
+
let headers = response?.schema?.fields?.map((field) => field.name || '') ?? [];
|
|
372
|
+
const headerTypes = response?.schema?.fields?.map((field) => String(field.type || 'STRING')) ?? [];
|
|
373
|
+
if (headers.length === 0 && rows.length > 0) {
|
|
374
|
+
headers = Object.keys(rows[0]);
|
|
375
|
+
}
|
|
376
|
+
return {
|
|
377
|
+
headers,
|
|
378
|
+
headerTypes: headerTypes.length > 0 ? headerTypes : undefined,
|
|
379
|
+
rows: rows.map((row) => headers.map((header) => normalizeValue(row[header]))),
|
|
380
|
+
totalRows: rows.length,
|
|
381
|
+
rowCount: rows.length,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
catch (error) {
|
|
385
|
+
if (isBigQueryTimeoutError(error)) {
|
|
386
|
+
throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
|
|
387
|
+
}
|
|
388
|
+
throw error;
|
|
314
389
|
}
|
|
315
|
-
return {
|
|
316
|
-
headers,
|
|
317
|
-
headerTypes: headerTypes.length > 0 ? headerTypes : undefined,
|
|
318
|
-
rows: rows.map((row) => headers.map((header) => normalizeValue(row[header]))),
|
|
319
|
-
totalRows: rows.length,
|
|
320
|
-
rowCount: rows.length,
|
|
321
|
-
};
|
|
322
390
|
}
|
|
323
391
|
async queryRaw(sql, params) {
|
|
324
392
|
const result = await this.query(sql, params);
|
|
@@ -328,14 +396,14 @@ export class KtxBigQueryScanConnector {
|
|
|
328
396
|
const rows = await this.queryRaw(sql);
|
|
329
397
|
return firstNumber(rows[0]?.[header]);
|
|
330
398
|
}
|
|
331
|
-
async introspectDataset(
|
|
399
|
+
async introspectDataset(ref, scopedNames, snapshotWarnings) {
|
|
332
400
|
if (scopedNames && scopedNames.length === 0)
|
|
333
401
|
return [];
|
|
334
|
-
const dataset = this.getClient().dataset(
|
|
402
|
+
const dataset = this.getClient().dataset(ref.dataset, ref.project);
|
|
335
403
|
const [tableRefs] = await dataset.getTables();
|
|
336
404
|
const scopeSet = scopedNames ? new Set(scopedNames) : null;
|
|
337
405
|
const filteredTableRefs = scopeSet ? tableRefs.filter((tableRef) => scopeSet.has(tableRef.id ?? '')) : tableRefs;
|
|
338
|
-
const primaryKeysResult = await tryConstraintQuery({ schema:
|
|
406
|
+
const primaryKeysResult = await tryConstraintQuery({ schema: ref.dataset, kind: 'primary_key', isDeniedError }, () => this.primaryKeys(ref));
|
|
339
407
|
const primaryKeys = primaryKeysResult.ok ? primaryKeysResult.value : new Map();
|
|
340
408
|
if (!primaryKeysResult.ok) {
|
|
341
409
|
snapshotWarnings.push(primaryKeysResult.warning);
|
|
@@ -343,39 +411,47 @@ export class KtxBigQueryScanConnector {
|
|
|
343
411
|
const tables = [];
|
|
344
412
|
for (const tableRef of filteredTableRefs) {
|
|
345
413
|
const tableName = tableRef.id || '';
|
|
346
|
-
const
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
414
|
+
const outcome = await tryIntrospectObject({ object: tableName, catalog: ref.project, db: ref.dataset }, async () => {
|
|
415
|
+
const [table] = await tableRef.get();
|
|
416
|
+
const fields = table.metadata.schema?.fields ?? [];
|
|
417
|
+
return {
|
|
418
|
+
catalog: ref.project,
|
|
419
|
+
db: ref.dataset,
|
|
420
|
+
name: tableName,
|
|
421
|
+
kind: tableKind(table.metadata.type),
|
|
422
|
+
comment: table.metadata.description || null,
|
|
423
|
+
estimatedRows: firstNumber(table.metadata.numRows) ?? 0,
|
|
424
|
+
columns: fields.map((field) => this.toSchemaColumn(tableName, field, primaryKeys)),
|
|
425
|
+
foreignKeys: [],
|
|
426
|
+
};
|
|
357
427
|
});
|
|
428
|
+
if (outcome.ok) {
|
|
429
|
+
tables.push(outcome.table);
|
|
430
|
+
}
|
|
431
|
+
else {
|
|
432
|
+
snapshotWarnings.push(outcome.warning);
|
|
433
|
+
}
|
|
358
434
|
}
|
|
359
435
|
return tables;
|
|
360
436
|
}
|
|
361
|
-
async primaryKeys(
|
|
437
|
+
async primaryKeys(ref) {
|
|
362
438
|
const rows = await this.queryRaw('SELECT tc.table_name, kcu.column_name ' +
|
|
363
439
|
'FROM `' +
|
|
364
|
-
|
|
440
|
+
ref.project +
|
|
365
441
|
'.' +
|
|
366
|
-
|
|
442
|
+
ref.dataset +
|
|
367
443
|
'.INFORMATION_SCHEMA.TABLE_CONSTRAINTS` tc ' +
|
|
368
444
|
'JOIN `' +
|
|
369
|
-
|
|
445
|
+
ref.project +
|
|
370
446
|
'.' +
|
|
371
|
-
|
|
447
|
+
ref.dataset +
|
|
372
448
|
'.INFORMATION_SCHEMA.KEY_COLUMN_USAGE` kcu ' +
|
|
373
449
|
'ON tc.constraint_name = kcu.constraint_name ' +
|
|
374
450
|
'AND tc.table_schema = kcu.table_schema ' +
|
|
375
451
|
'AND tc.table_name = kcu.table_name ' +
|
|
376
452
|
"WHERE tc.constraint_type = 'PRIMARY KEY' " +
|
|
377
453
|
"AND tc.table_schema = '" +
|
|
378
|
-
|
|
454
|
+
ref.dataset +
|
|
379
455
|
"' " +
|
|
380
456
|
"AND NOT REGEXP_CONTAINS(kcu.column_name, r'^(stacksync_record_id|sync_primary_key)_') " +
|
|
381
457
|
'ORDER BY tc.table_name, kcu.ordinal_position');
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { KtxSqlDialect } from '../../context/connections/dialects.js';
|
|
2
2
|
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
|
3
3
|
type BigQueryTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
|
4
4
|
/** @internal */
|
|
5
|
-
export declare class KtxBigQueryDialect implements
|
|
5
|
+
export declare class KtxBigQueryDialect implements KtxSqlDialect {
|
|
6
6
|
readonly type: "bigquery";
|
|
7
7
|
private readonly typeMappings;
|
|
8
8
|
quoteIdentifier(identifier: string): string;
|
|
@@ -90,6 +90,7 @@ export declare class KtxClickHouseScanConnector implements KtxScanConnector {
|
|
|
90
90
|
private readonly clientFactory;
|
|
91
91
|
private readonly endpointResolver?;
|
|
92
92
|
private readonly now;
|
|
93
|
+
private readonly deadlineMs;
|
|
93
94
|
private readonly dialect;
|
|
94
95
|
private client;
|
|
95
96
|
private resolvedEndpoint;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
|
-
import {
|
|
2
|
+
import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
|
|
3
|
+
import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
|
|
3
4
|
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
|
4
5
|
import { connectorTestFailure, createKtxConnectorCapabilities } from '../../context/scan/types.js';
|
|
5
6
|
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
|
@@ -17,6 +18,20 @@ function stringConfigValue(connection, key, env) {
|
|
|
17
18
|
function maybeNumber(value) {
|
|
18
19
|
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
19
20
|
}
|
|
21
|
+
// ClickHouse error code 159 = TIMEOUT_EXCEEDED, raised when max_execution_time
|
|
22
|
+
// is hit. The client surfaces it via a numeric/string `code` or a "Code: 159"
|
|
23
|
+
// message prefix depending on transport.
|
|
24
|
+
function isClickHouseTimeoutError(error) {
|
|
25
|
+
if (!error || typeof error !== 'object') {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
const code = error.code;
|
|
29
|
+
if (code === 159 || code === '159') {
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
const message = error.message;
|
|
33
|
+
return typeof message === 'string' && (/\bCode:\s*159\b/.test(message) || message.includes('TIMEOUT_EXCEEDED'));
|
|
34
|
+
}
|
|
20
35
|
function parseClickHouseUrl(url) {
|
|
21
36
|
const parsed = new URL(url);
|
|
22
37
|
return {
|
|
@@ -126,7 +141,8 @@ export class KtxClickHouseScanConnector {
|
|
|
126
141
|
clientFactory;
|
|
127
142
|
endpointResolver;
|
|
128
143
|
now;
|
|
129
|
-
|
|
144
|
+
deadlineMs;
|
|
145
|
+
dialect = getSqlDialectForDriver('clickhouse');
|
|
130
146
|
client = null;
|
|
131
147
|
resolvedEndpoint = null;
|
|
132
148
|
constructor(options) {
|
|
@@ -140,6 +156,7 @@ export class KtxClickHouseScanConnector {
|
|
|
140
156
|
this.clientFactory = options.clientFactory ?? new DefaultClickHouseClientFactory();
|
|
141
157
|
this.endpointResolver = options.endpointResolver;
|
|
142
158
|
this.now = options.now ?? (() => new Date());
|
|
159
|
+
this.deadlineMs = resolveQueryDeadlineMs(this.connection);
|
|
143
160
|
this.id = `clickhouse:${options.connectionId}`;
|
|
144
161
|
}
|
|
145
162
|
async testConnection() {
|
|
@@ -375,9 +392,13 @@ export class KtxClickHouseScanConnector {
|
|
|
375
392
|
username: config.username,
|
|
376
393
|
password: config.password ?? '',
|
|
377
394
|
database: config.database,
|
|
378
|
-
request_timeout
|
|
395
|
+
// The server aborts at max_execution_time (seconds); request_timeout must
|
|
396
|
+
// outlast it so the HTTP client receives the code-159 error instead of
|
|
397
|
+
// giving up first and leaving the query running.
|
|
398
|
+
request_timeout: this.deadlineMs + 5_000,
|
|
379
399
|
clickhouse_settings: {
|
|
380
400
|
output_format_json_quote_64bit_integers: 1,
|
|
401
|
+
max_execution_time: Math.ceil(this.deadlineMs / 1000),
|
|
381
402
|
},
|
|
382
403
|
...(isProxied && config.ssl
|
|
383
404
|
? {
|
|
@@ -402,19 +423,27 @@ export class KtxClickHouseScanConnector {
|
|
|
402
423
|
}
|
|
403
424
|
async query(sql, params) {
|
|
404
425
|
const client = await this.clientForQuery();
|
|
405
|
-
|
|
406
|
-
query
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
426
|
+
try {
|
|
427
|
+
const resultSet = await client.query({
|
|
428
|
+
query: assertReadOnlySql(sql),
|
|
429
|
+
format: 'JSONCompact',
|
|
430
|
+
...(params ? { query_params: params } : {}),
|
|
431
|
+
});
|
|
432
|
+
const response = (await resultSet.json());
|
|
433
|
+
const meta = response.meta ?? [];
|
|
434
|
+
return {
|
|
435
|
+
headers: meta.map((field) => field.name),
|
|
436
|
+
headerTypes: meta.map((field) => field.type),
|
|
437
|
+
rows: response.data ?? [],
|
|
438
|
+
totalRows: response.rows ?? response.data?.length ?? 0,
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
catch (error) {
|
|
442
|
+
if (isClickHouseTimeoutError(error)) {
|
|
443
|
+
throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
|
|
444
|
+
}
|
|
445
|
+
throw error;
|
|
446
|
+
}
|
|
418
447
|
}
|
|
419
448
|
assertConnection(connectionId) {
|
|
420
449
|
if (connectionId !== this.connectionId) {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { KtxSqlDialect } from '../../context/connections/dialects.js';
|
|
2
2
|
import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
|
|
3
3
|
type ClickHouseTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
|
|
4
4
|
/** @internal */
|
|
5
|
-
export declare class KtxClickHouseDialect implements
|
|
5
|
+
export declare class KtxClickHouseDialect implements KtxSqlDialect {
|
|
6
6
|
readonly type: "clickhouse";
|
|
7
7
|
private readonly typeMappings;
|
|
8
8
|
quoteIdentifier(identifier: string): string;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxConnectorTestResult, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaSnapshot, type KtxTableListEntry, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
|
2
|
+
import { type KtxMongoDocument } from './schema-inference.js';
|
|
3
|
+
export interface KtxMongoDbConnectionConfig {
|
|
4
|
+
driver?: string;
|
|
5
|
+
url?: string;
|
|
6
|
+
database?: string;
|
|
7
|
+
databases?: string[];
|
|
8
|
+
enabled_tables?: string[];
|
|
9
|
+
sample_size?: number;
|
|
10
|
+
order_by?: string;
|
|
11
|
+
[key: string]: unknown;
|
|
12
|
+
}
|
|
13
|
+
export interface KtxMongoListedCollection {
|
|
14
|
+
name: string;
|
|
15
|
+
type?: string;
|
|
16
|
+
}
|
|
17
|
+
interface KtxMongoFindOptions {
|
|
18
|
+
sort: Record<string, 1 | -1>;
|
|
19
|
+
limit: number;
|
|
20
|
+
projection?: Record<string, 1>;
|
|
21
|
+
}
|
|
22
|
+
/** Driver-agnostic seam over the `mongodb` client so the connector is unit-testable without a server. */
|
|
23
|
+
export interface KtxMongoClient {
|
|
24
|
+
listCollections(databaseName: string): Promise<KtxMongoListedCollection[]>;
|
|
25
|
+
estimatedDocumentCount(databaseName: string, collectionName: string): Promise<number>;
|
|
26
|
+
find(databaseName: string, collectionName: string, options: KtxMongoFindOptions): Promise<KtxMongoDocument[]>;
|
|
27
|
+
ping(databaseName: string): Promise<void>;
|
|
28
|
+
close(): Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
export interface KtxMongoClientFactory {
|
|
31
|
+
create(url: string): KtxMongoClient;
|
|
32
|
+
}
|
|
33
|
+
export interface KtxMongoDbScanConnectorOptions {
|
|
34
|
+
connectionId: string;
|
|
35
|
+
connection: KtxMongoDbConnectionConfig | undefined;
|
|
36
|
+
clientFactory?: KtxMongoClientFactory;
|
|
37
|
+
env?: NodeJS.ProcessEnv;
|
|
38
|
+
now?: () => Date;
|
|
39
|
+
}
|
|
40
|
+
export declare function isKtxMongoDbConnectionConfig(connection: KtxMongoDbConnectionConfig | undefined): connection is KtxMongoDbConnectionConfig;
|
|
41
|
+
export declare class KtxMongoDbScanConnector implements KtxScanConnector {
|
|
42
|
+
readonly id: string;
|
|
43
|
+
readonly driver: "mongodb";
|
|
44
|
+
readonly capabilities: import("../../context/scan/types.js").KtxConnectorCapabilities;
|
|
45
|
+
private readonly connectionId;
|
|
46
|
+
private readonly connection;
|
|
47
|
+
private readonly url;
|
|
48
|
+
private readonly databases;
|
|
49
|
+
private readonly sampleSize;
|
|
50
|
+
private readonly orderBy;
|
|
51
|
+
private readonly enabledTables;
|
|
52
|
+
private readonly clientFactory;
|
|
53
|
+
private readonly now;
|
|
54
|
+
private readonly dialect;
|
|
55
|
+
private client;
|
|
56
|
+
constructor(options: KtxMongoDbScanConnectorOptions);
|
|
57
|
+
testConnection(): Promise<KtxConnectorTestResult>;
|
|
58
|
+
introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot>;
|
|
59
|
+
private introspectCollection;
|
|
60
|
+
sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise<KtxTableSampleResult>;
|
|
61
|
+
sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise<KtxColumnSampleResult>;
|
|
62
|
+
listSchemas(): Promise<string[]>;
|
|
63
|
+
listTables(schemas?: string[]): Promise<KtxTableListEntry[]>;
|
|
64
|
+
cleanup(): Promise<void>;
|
|
65
|
+
private resolveTableRef;
|
|
66
|
+
private clientForQuery;
|
|
67
|
+
private assertConnection;
|
|
68
|
+
}
|
|
69
|
+
export {};
|