@yolean/evidence-sdk 0.0.0-stage → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +544 -0
- package/LICENSE.md +9 -0
- package/README.md +237 -2
- package/package.json +161 -3
- package/src/build-dev/cli/add/addServerHook.js +30 -0
- package/src/build-dev/cli/add/addTypeRef.js +19 -0
- package/src/build-dev/cli/add/updateGitIgnore.js +33 -0
- package/src/build-dev/cli/add/updatePackageJson.js +26 -0
- package/src/build-dev/cli/add/updateSvelteConfig.js +199 -0
- package/src/build-dev/cli/add/updateViteConfig.js +152 -0
- package/src/build-dev/cli/add.js +95 -0
- package/src/build-dev/cli/build.js +30 -0
- package/src/build-dev/cli/dev.js +49 -0
- package/src/build-dev/cli/index.js +14 -0
- package/src/build-dev/cli/upgrade.js +54 -0
- package/src/build-dev/svelte/index.js +54 -0
- package/src/build-dev/svelte/lib/isTargetFile.js +6 -0
- package/src/build-dev/svelte/metadatas.js +39 -0
- package/src/build-dev/svelte/processors/addBasePathToHrefAndSrc/addBasePathToHrefAndSrc.js +29 -0
- package/src/build-dev/svelte/processors/addBasePathToHrefAndSrc/addBasePathToHrefAndSrc.spec.js +82 -0
- package/src/build-dev/svelte/processors/addScriptTags.js +17 -0
- package/src/build-dev/svelte/processors/convertToHtml/convertToHtml.js +19 -0
- package/src/build-dev/svelte/processors/convertToHtml/convertToHtml.spec.js +123 -0
- package/src/build-dev/svelte/processors/convertToHtml/processMarkdown.js +59 -0
- package/src/build-dev/svelte/processors/extractQueries/extractQueries.js +49 -0
- package/src/build-dev/svelte/processors/extractQueries/extractQueries.spec.js +54 -0
- package/src/build-dev/svelte/processors/highlight/highlight.js +52 -0
- package/src/build-dev/svelte/processors/highlight/highlight.spec.js +31 -0
- package/src/build-dev/svelte/processors/initializeUniversalSql.js +15 -0
- package/src/build-dev/svelte/processors/injectComponents/injectComponents.js +56 -0
- package/src/build-dev/svelte/processors/injectHighlightStyles.js +14 -0
- package/src/build-dev/svelte/processors/injectQueries/injectQueries.js +77 -0
- package/src/build-dev/svelte/processors/markRootLayoutPrerenderable.js +56 -0
- package/src/build-dev/svelte/processors/obfuscateCode/obfuscateCode.js +17 -0
- package/src/build-dev/svelte/processors/obfuscateCode/obfuscateCode.spec.js +21 -0
- package/src/build-dev/svelte/processors/transformQueries/transformQueries.js +58 -0
- package/src/build-dev/vite/config-virtual.js +24 -0
- package/src/build-dev/vite/constants.js +7 -0
- package/src/build-dev/vite/datasource-plugins/source-query-hmr.js +117 -0
- package/src/build-dev/vite/evidence-config.d.ts +6 -0
- package/src/build-dev/vite/index.js +214 -0
- package/src/build-dev/vite/layout-plugins/copy-to-layout.js +71 -0
- package/src/build-dev/vite/plugins.js +4 -0
- package/src/build-dev/vite/query-directory-hmr.js +32 -0
- package/src/build-dev/vite/sidecar-app.js +79 -0
- package/src/build-dev/vite/virtuals/QuerySSR.svelte +13 -0
- package/src/build-dev/vite/virtuals/__tests/proxyStack.spec.js +52 -0
- package/src/build-dev/vite/virtuals/bootstrapContexts.svelte.js +27 -0
- package/src/build-dev/vite/virtuals/browser/static-assets.js +3 -0
- package/src/build-dev/vite/virtuals/contextKeys.js +3 -0
- package/src/build-dev/vite/virtuals/initUsql.js +23 -0
- package/src/build-dev/vite/virtuals/node/static-assets.js +56 -0
- package/src/build-dev/vite/virtuals/proxyStack.js +37 -0
- package/src/build-dev/vite/virtuals/queries.svelte.js +150 -0
- package/src/build-dev/vite/virtuals/ssrHook.svelte.js +60 -0
- package/src/build-dev/vite/virtuals/updateQueryContext.svelte.js +31 -0
- package/src/build-dev/vite/virtuals-dynamic.js +12 -0
- package/src/cli.js +5 -0
- package/src/cli.root.js +90 -0
- package/src/configuration/cli/create.js +20 -0
- package/src/configuration/cli/index.js +12 -0
- package/src/configuration/createEvidenceConfig.js +51 -0
- package/src/configuration/getEvidenceConfig.js +60 -0
- package/src/configuration/getEvidenceConfig.legacy.js +27 -0
- package/src/configuration/index.js +1 -0
- package/src/configuration/schemas/config.schema.js +11 -0
- package/src/configuration/schemas/deployment.schema.js +18 -0
- package/src/configuration/schemas/deployment.schema.spec.js +34 -0
- package/src/configuration/updateEvidenceConfig.js +25 -0
- package/src/legacy-compat.js +8 -0
- package/src/lib/EvidenceError.js +35 -0
- package/src/lib/b64Deep.js +55 -0
- package/src/lib/check-import.js +12 -0
- package/src/lib/cleanZodErrors.js +22 -0
- package/src/lib/cli/cmdFail.js +33 -0
- package/src/lib/cli/notices.js +44 -0
- package/src/lib/debug.js +40 -0
- package/src/lib/flows.d.ts +21 -0
- package/src/lib/forceLink.js +28 -0
- package/src/lib/history/index.js +70 -0
- package/src/lib/history/types.d.ts +8 -0
- package/src/lib/pkg-types.helpers.js +16 -0
- package/src/lib/processing-queue.js +96 -0
- package/src/lib/processing-queue.spec.js +148 -0
- package/src/lib/projectPaths.js +29 -0
- package/src/lib/projectRoot.js +12 -0
- package/src/lib/sharedPromise.js +77 -0
- package/src/lib/sharedPromise.spec.js +23 -0
- package/src/lib/strict.js +19 -0
- package/src/lib/tests/fs.js +21 -0
- package/src/lib/tokens.js +7 -0
- package/src/lib/unnest-zod-error.js +14 -0
- package/src/logger/EvidenceLogger.js +164 -0
- package/src/logger/index.js +4 -0
- package/src/meta.js +8 -0
- package/src/plugins/cli/index.js +13 -0
- package/src/plugins/cli/install.js +26 -0
- package/src/plugins/cli/install.prompt.js +121 -0
- package/src/plugins/components/getAllComponents.js +82 -0
- package/src/plugins/components/getComponentsInPlugin.js +33 -0
- package/src/plugins/components/loadComponentPlugins.js +48 -0
- package/src/plugins/components/loaders/file-loader.js +125 -0
- package/src/plugins/components/loaders/manifest-loader.js +53 -0
- package/src/plugins/components/schemas/component-manifest.schema.js +5 -0
- package/src/plugins/components/types.d.ts +21 -0
- package/src/plugins/components/validateOverrides.js +30 -0
- package/src/plugins/components/validateOverrides.spec.js +32 -0
- package/src/plugins/datasources/Datasources.js +85 -0
- package/src/plugins/datasources/Datasources.spec.js +78 -0
- package/src/plugins/datasources/SourceResultCache.js +62 -0
- package/src/plugins/datasources/buildSourceDirectoryProxy.js +21 -0
- package/src/plugins/datasources/cli/edit/Options.fixtures.js +69 -0
- package/src/plugins/datasources/cli/edit/Options.js +253 -0
- package/src/plugins/datasources/cli/edit/Options.spec.js +496 -0
- package/src/plugins/datasources/cli/edit/editSourceConfig.js +201 -0
- package/src/plugins/datasources/cli/edit.js +20 -0
- package/src/plugins/datasources/cli/edit.prompt.js +100 -0
- package/src/plugins/datasources/cli/index.js +18 -0
- package/src/plugins/datasources/cli/sources.js +77 -0
- package/src/plugins/datasources/evalSources.js +257 -0
- package/src/plugins/datasources/getPublishedDatasources.js +62 -0
- package/src/plugins/datasources/index.js +6 -0
- package/src/plugins/datasources/loadSourceConfig.js +147 -0
- package/src/plugins/datasources/loadSourceConfig.spec.js +127 -0
- package/src/plugins/datasources/loadSourcePlugins.js +49 -0
- package/src/plugins/datasources/loadSources.js +48 -0
- package/src/plugins/datasources/schemas/datasource.schema.js +52 -0
- package/src/plugins/datasources/schemas/datasourceCache.schema.js +3 -0
- package/src/plugins/datasources/schemas/datasourcePlugin.schema.js +37 -0
- package/src/plugins/datasources/schemas/datasourcePluginOptions.schema.js +64 -0
- package/src/plugins/datasources/schemas/index.js +2 -0
- package/src/plugins/datasources/schemas/manifest.schema.js +8 -0
- package/src/plugins/datasources/schemas/results.schema.js +126 -0
- package/src/plugins/datasources/sub-source-vars.js +55 -0
- package/src/plugins/datasources/sub-source-vars.spec.js +46 -0
- package/src/plugins/datasources/types.d.ts +53 -0
- package/src/plugins/datasources/updateManifest.js +115 -0
- package/src/plugins/datasources/updateManifest.spec.js +200 -0
- package/src/plugins/datasources/wrapSimpleConnector.js +95 -0
- package/src/plugins/datasources/writeSourceConfig.js +60 -0
- package/src/plugins/index.js +2 -0
- package/src/plugins/layouts/copyMethods/index.js +10 -0
- package/src/plugins/layouts/copyMethods/svelte.js +53 -0
- package/src/plugins/layouts/copyMethods/sveltekit.js +75 -0
- package/src/plugins/layouts/copyToLayout.js +106 -0
- package/src/plugins/layouts/loadLayoutPlugin.js +23 -0
- package/src/plugins/loadPluginPackage.js +77 -0
- package/src/plugins/schemas/index.js +1 -0
- package/src/plugins/schemas/plugin-config.schema.js +41 -0
- package/src/plugins/schemas/plugin-config.schema.spec.js +21 -0
- package/src/plugins/schemas/plugin-package.schema.js +154 -0
- package/src/plugins/schemas/plugin-package.schema.spec.js +30 -0
- package/src/query-builder.d.ts +1 -0
- package/src/query-builder.js +1 -0
- package/src/types/duckdb-wellknown.d.ts +35 -0
- package/src/types/html-attributes.ts +13 -0
- package/src/types/index.d.ts +18 -0
- package/src/types/mosaic-sql.d.ts +27 -0
- package/src/types/sveltekit-autoimport.d.ts +13 -0
- package/src/types/virtuals.d.ts +113 -0
- package/src/types.d.ts +1 -0
- package/src/usql/index.d.ts +4 -0
- package/src/usql/index.js +3 -0
- package/src/usql/query/Query.js +1391 -0
- package/src/usql/query/Query.spec.js +827 -0
- package/src/usql/query/queryScore.js +92 -0
- package/src/usql/query/queryScore.spec.js +26 -0
- package/src/usql/query/sterilizeQuery.js +89 -0
- package/src/usql/query/sterilizeQuery.spec.js +172 -0
- package/src/usql/setTrackProxy/setTrackProxy.js +80 -0
- package/src/usql/setTrackProxy/setTrackProxy.spec.js +166 -0
- package/src/usql/types.ts +86 -0
- package/src/usql/utilities/duckdbSerialize.js +29 -0
- package/src/usql/utilities/index.js +2 -0
- package/src/usql/utilities/resolveMaybePromise.js +30 -0
- package/src/utils/batchUp.js +22 -0
- package/src/utils/batchUp.spec.js +69 -0
- package/src/utils/browserDebounce.js +16 -0
- package/src/utils/index.js +7 -0
- package/src/utils/svelte/addBasePath.js +74 -0
- package/src/utils/svelte/addBasePath.spec.js +108 -0
- package/src/utils/svelte/index.js +13 -0
- package/src/utils/svelte/inputs.js +138 -0
- package/tsconfig.json +22 -0
- package/vite.config.js +3 -0
- package/vitest.config.js +15 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import yaml from 'yaml';
|
|
3
|
+
import fs from 'fs/promises';
|
|
4
|
+
|
|
5
|
+
import { loadSourcePlugins } from '../loadSourcePlugins.js';
|
|
6
|
+
import { loadSources } from '../loadSources.js';
|
|
7
|
+
import * as prompt from '@clack/prompts';
|
|
8
|
+
import { EvidenceError } from '../../../lib/EvidenceError.js';
|
|
9
|
+
import { editSourceConfig } from './edit/editSourceConfig.js';
|
|
10
|
+
|
|
11
|
+
export const editConnectionsPrompt = async () => {
|
|
12
|
+
const dataSpinner = prompt.spinner();
|
|
13
|
+
|
|
14
|
+
dataSpinner.start('Fetching connections and plugins');
|
|
15
|
+
const [sources, plugins] = await Promise.all([loadSources(), loadSourcePlugins()]);
|
|
16
|
+
dataSpinner.stop('Done!', 0);
|
|
17
|
+
|
|
18
|
+
const selectedSourceName = await prompt.select({
|
|
19
|
+
message: 'Which source are we editing?',
|
|
20
|
+
options: [
|
|
21
|
+
...sources.map((source) => ({
|
|
22
|
+
value: source.name,
|
|
23
|
+
label: `${source.name} (${source.type})`
|
|
24
|
+
})),
|
|
25
|
+
{
|
|
26
|
+
value: null,
|
|
27
|
+
label: 'Create new connection'
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
if (selectedSourceName) {
|
|
33
|
+
const selectedSource = sources.find((source) => source.name === selectedSourceName);
|
|
34
|
+
if (!selectedSource) throw new EvidenceError('');
|
|
35
|
+
const plugin = plugins.getBySource(selectedSource.type);
|
|
36
|
+
if (!plugin)
|
|
37
|
+
throw new EvidenceError('Tried to edit source that does not have a correlated plugin');
|
|
38
|
+
await editSourceConfig(selectedSource, plugin[1]);
|
|
39
|
+
} else {
|
|
40
|
+
const canonSources = plugins.getCanonicalSources();
|
|
41
|
+
|
|
42
|
+
if (!canonSources.length) {
|
|
43
|
+
prompt.log.error(
|
|
44
|
+
'No source plugins are installed, or none were found in evidence.config.yaml'
|
|
45
|
+
);
|
|
46
|
+
prompt.cancel('Operation Failed');
|
|
47
|
+
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const selectedType = await prompt.select({
|
|
52
|
+
message: 'What source should we connect to?',
|
|
53
|
+
options: canonSources.map((s) => ({ value: s }))
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
if (prompt.isCancel(selectedType)) {
|
|
57
|
+
prompt.cancel('Did not create connection!');
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** @type {string | symbol} */
|
|
62
|
+
const name = await prompt
|
|
63
|
+
.text({
|
|
64
|
+
message: 'What should we call this connection?',
|
|
65
|
+
validate: (s) => {
|
|
66
|
+
if (s.length === 0) return 'Connection name must not be empty';
|
|
67
|
+
if (!/^[\w_]+$/.test(s))
|
|
68
|
+
return 'Connection name can only contain letters, numbers, and underscores';
|
|
69
|
+
}
|
|
70
|
+
})
|
|
71
|
+
.then((r) => (prompt.isCancel(r) ? r : r?.toString()));
|
|
72
|
+
|
|
73
|
+
if (prompt.isCancel(name)) {
|
|
74
|
+
prompt.cancel('Did not create connection!');
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const existingSourceDirs = await fs.readdir(path.resolve('sources'));
|
|
79
|
+
if (existingSourceDirs.includes(name))
|
|
80
|
+
throw new EvidenceError(
|
|
81
|
+
`Cannot create source ${name}, directory ./sources/${name} already exists`
|
|
82
|
+
);
|
|
83
|
+
const sourceWithSameName = sources.find((source) => source.name === name);
|
|
84
|
+
if (sourceWithSameName)
|
|
85
|
+
throw new EvidenceError(
|
|
86
|
+
`Cannot create source ${name}, a source with that name already exists in ${sourceWithSameName.dir}`
|
|
87
|
+
);
|
|
88
|
+
await fs.mkdir(path.join(path.resolve('sources'), name));
|
|
89
|
+
const comment = `# This file was automatically generated\n`;
|
|
90
|
+
await fs.writeFile(
|
|
91
|
+
path.join(path.resolve('sources'), name, 'connection.yaml'),
|
|
92
|
+
comment + yaml.stringify({ name, type: selectedType })
|
|
93
|
+
);
|
|
94
|
+
|
|
95
|
+
const newSource = (await loadSources()).find((s) => s.name === name);
|
|
96
|
+
if (!newSource)
|
|
97
|
+
throw new EvidenceError('Internal Error', ['Failed to load just-created source']);
|
|
98
|
+
await editSourceConfig(newSource, plugins.getBySource(newSource.type)[1]);
|
|
99
|
+
}
|
|
100
|
+
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { sources } from './sources.js';
|
|
2
|
+
import { edit } from './edit.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @type {import("@brianmd/citty").SubCommandsDef}
|
|
6
|
+
*/
|
|
7
|
+
export const datasourcesCli = {
|
|
8
|
+
sources,
|
|
9
|
+
connections: {
|
|
10
|
+
meta: {
|
|
11
|
+
name: 'connections',
|
|
12
|
+
description: 'Manage your source connections'
|
|
13
|
+
},
|
|
14
|
+
subCommands: {
|
|
15
|
+
edit
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import { dataDirectory, metaDirectory } from '../../../lib/projectPaths.js';
|
|
3
|
+
import { evalSources } from '../evalSources.js';
|
|
4
|
+
import { updateManifest } from '../updateManifest.js';
|
|
5
|
+
import { log } from '../../../logger/index.js';
|
|
6
|
+
import { enableStrictMode, isStrictMode } from '../../../lib/strict.js';
|
|
7
|
+
import { EvidenceError } from '../../../lib/EvidenceError.js';
|
|
8
|
+
|
|
9
|
+
/** @type { import("@brianmd/citty").CommandDef } */
|
|
10
|
+
export const sources = {
|
|
11
|
+
meta: {
|
|
12
|
+
name: 'sources',
|
|
13
|
+
description: 'Update your Evidence source outputs'
|
|
14
|
+
},
|
|
15
|
+
|
|
16
|
+
args: {
|
|
17
|
+
changed: {
|
|
18
|
+
type: 'boolean',
|
|
19
|
+
description: 'Run only queries that have changed',
|
|
20
|
+
default: false
|
|
21
|
+
},
|
|
22
|
+
sources: {
|
|
23
|
+
type: 'string',
|
|
24
|
+
description: "Run only the listed sources (',' delimited)"
|
|
25
|
+
},
|
|
26
|
+
queries: {
|
|
27
|
+
type: 'string',
|
|
28
|
+
description: "Run only the listed queries (',' delimited)"
|
|
29
|
+
},
|
|
30
|
+
strict: {
|
|
31
|
+
type: 'boolean',
|
|
32
|
+
description: 'Fail when a source query fails',
|
|
33
|
+
default: false
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
|
|
37
|
+
async run({ args }) {
|
|
38
|
+
if (args.strict) {
|
|
39
|
+
enableStrictMode();
|
|
40
|
+
if (!isStrictMode()) {
|
|
41
|
+
throw new EvidenceError('Failed to enable strict mode');
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const sources = new Set(
|
|
46
|
+
args.sources
|
|
47
|
+
?.toString()
|
|
48
|
+
.split(',')
|
|
49
|
+
.map(/** @param {string} s */ (s) => s.trim())
|
|
50
|
+
);
|
|
51
|
+
if (sources.size) log.debug(`\tRunning sources: ${[...sources].join(', ')}`);
|
|
52
|
+
|
|
53
|
+
const queries = new Set(
|
|
54
|
+
args.queries
|
|
55
|
+
?.toString()
|
|
56
|
+
.split(',')
|
|
57
|
+
.map(/** @param {string} s */ (s) => s.trim())
|
|
58
|
+
);
|
|
59
|
+
if (queries.size) log.debug(`\tRunning queries: ${[...queries].join(', ')}`);
|
|
60
|
+
|
|
61
|
+
const evaluatedManifest = await evalSources(
|
|
62
|
+
dataDirectory,
|
|
63
|
+
metaDirectory,
|
|
64
|
+
{
|
|
65
|
+
sources,
|
|
66
|
+
queries,
|
|
67
|
+
only_changed: args.changed === true
|
|
68
|
+
},
|
|
69
|
+
isStrictMode()
|
|
70
|
+
);
|
|
71
|
+
|
|
72
|
+
log.info(' Evaluated sources, saving manifest');
|
|
73
|
+
|
|
74
|
+
await updateManifest(evaluatedManifest, dataDirectory);
|
|
75
|
+
log.info(chalk.bold.green(' ✅ Done!'));
|
|
76
|
+
}
|
|
77
|
+
};
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import fs from 'fs/promises';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
|
|
5
|
+
import { EvidenceError } from '../../lib/EvidenceError.js';
|
|
6
|
+
import { loadSourcePlugins } from './loadSourcePlugins.js';
|
|
7
|
+
import { loadSources } from './loadSources.js';
|
|
8
|
+
import { wrapSimpleConnector } from './wrapSimpleConnector.js';
|
|
9
|
+
import { buildMultipartParquet } from '@evidence-dev/universal-sql';
|
|
10
|
+
import { buildSourceDirectoryProxy } from './buildSourceDirectoryProxy.js';
|
|
11
|
+
import { addToCache, checkCache, flushCache, loadCache } from './SourceResultCache.js';
|
|
12
|
+
import ora from 'ora';
|
|
13
|
+
import { dataUrlPrefix } from '../../lib/projectPaths.js';
|
|
14
|
+
import { subSourceVariables } from './sub-source-vars.js';
|
|
15
|
+
import { logQueryEvent } from '@evidence-dev/telemetry';
|
|
16
|
+
|
|
17
|
+
// TODO: This is a great candidate for unit testing - but it may need to be broken down further to make that more achievable, right now it would take a _lot_ of mocks
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {string} dataPath
|
|
21
|
+
* @param {string} metaPath
|
|
22
|
+
* @param {import('./types.js').SourceFilters} [filters] `sources` or `queries` being null means no filter
|
|
23
|
+
* @param {boolean} [strict]
|
|
24
|
+
* @returns {Promise<import('./types.js').Manifest>}
|
|
25
|
+
*/
|
|
26
|
+
export const evalSources = async (dataPath, metaPath, filters, strict) => {
|
|
27
|
+
const pluginLoader = ora({ text: 'Loading plugins & sources' }).start();
|
|
28
|
+
|
|
29
|
+
// Setup work
|
|
30
|
+
const [sourcePlugins, sources] = await Promise.all([
|
|
31
|
+
loadSourcePlugins(),
|
|
32
|
+
loadSources(pluginLoader),
|
|
33
|
+
loadCache(metaPath)
|
|
34
|
+
]).catch((e) => {
|
|
35
|
+
pluginLoader.fail();
|
|
36
|
+
throw e;
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
if (sources.length) {
|
|
40
|
+
pluginLoader.succeed();
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** @type {import('./types.js').Manifest} */
|
|
44
|
+
const outputManifest = { renderedFiles: {}, locatedFiles: {}, locatedSchemas: [] };
|
|
45
|
+
|
|
46
|
+
/** @type {string[]} */
|
|
47
|
+
const skippedSources = [];
|
|
48
|
+
|
|
49
|
+
for (const source of sources) {
|
|
50
|
+
outputManifest.locatedSchemas ??= [];
|
|
51
|
+
outputManifest.locatedSchemas.push(source.name);
|
|
52
|
+
if (filters?.sources?.size && !filters?.sources?.has(source.name)) {
|
|
53
|
+
console.debug(` [Skipping]: ${chalk.bold(source.name)}`);
|
|
54
|
+
skippedSources.push(source.name);
|
|
55
|
+
continue;
|
|
56
|
+
} else {
|
|
57
|
+
console.log(chalk.dim('-'.repeat(5)));
|
|
58
|
+
console.log(chalk.green(` [Processing] ${chalk.bold(source.name)}`));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const plugin = sourcePlugins.getBySource(source.type);
|
|
62
|
+
if (!plugin) {
|
|
63
|
+
logQueryEvent('source-connector-not-found', source.type, source.name);
|
|
64
|
+
// TODO: How forgiving do we want to be here?
|
|
65
|
+
// TODO: If we want to be really fancy; we could batch these, and do an NPM lookup at the end to say "these packages provide those datasources"
|
|
66
|
+
throw new EvidenceError(
|
|
67
|
+
`Could not find matching datasource plugin for ${chalk.bold(
|
|
68
|
+
source.name
|
|
69
|
+
)} (source: ${chalk.bold(source.type)})`
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
const [, mod] = plugin;
|
|
73
|
+
const testResult = await mod.testConnection(source.options, source.dir);
|
|
74
|
+
if (testResult !== true) {
|
|
75
|
+
logQueryEvent('db-connection-error', source.type, source.name);
|
|
76
|
+
throw new EvidenceError(
|
|
77
|
+
`Error connecting to datasource ${chalk.bold(source.name)}: ${testResult.reason}`
|
|
78
|
+
);
|
|
79
|
+
} else {
|
|
80
|
+
logQueryEvent('db-connection-success', source.type, source.name);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** @type {import('./types.js').ProcessSourceFn} */
|
|
84
|
+
const tableIter = 'processSource' in mod ? mod.processSource : wrapSimpleConnector(mod, source);
|
|
85
|
+
|
|
86
|
+
const utils = buildUtils(source, filters);
|
|
87
|
+
|
|
88
|
+
outputManifest.renderedFiles[source.name] = [];
|
|
89
|
+
if (!outputManifest.locatedFiles) outputManifest.locatedFiles = {};
|
|
90
|
+
outputManifest.locatedFiles[source.name] = [];
|
|
91
|
+
|
|
92
|
+
const iter = tableIter(source.options, await buildSourceDirectoryProxy(source.dir), utils)[
|
|
93
|
+
Symbol.asyncIterator
|
|
94
|
+
](); // this is required for typescript to be happy
|
|
95
|
+
|
|
96
|
+
/** @type {IteratorResult<import('./types.js').QueryResultTable<any> | EvidenceError>} */
|
|
97
|
+
let iterResult;
|
|
98
|
+
|
|
99
|
+
while (((iterResult = await iter.next()), !iterResult.done)) {
|
|
100
|
+
if (iterResult.done) continue;
|
|
101
|
+
if (iterResult.value instanceof Error) {
|
|
102
|
+
const error = iterResult.value;
|
|
103
|
+
let tableName = 'Unknown';
|
|
104
|
+
if (error instanceof EvidenceError) {
|
|
105
|
+
if (error.metadata.tableName) tableName = error.metadata.tableName;
|
|
106
|
+
}
|
|
107
|
+
ora({
|
|
108
|
+
prefixText: ` ${tableName}`,
|
|
109
|
+
spinner: 'triangle',
|
|
110
|
+
discardStdin: false,
|
|
111
|
+
interval: 250
|
|
112
|
+
}).fail(`Error: ${error.message}`);
|
|
113
|
+
logQueryEvent('db-error', source.type, source.name, tableName);
|
|
114
|
+
if (strict) throw error;
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
const table = { ...iterResult.value };
|
|
118
|
+
const spinner = ora({
|
|
119
|
+
prefixText: ` ${table.name}`,
|
|
120
|
+
spinner: 'triangle',
|
|
121
|
+
discardStdin: false,
|
|
122
|
+
interval: 250
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
outputManifest.locatedFiles[source.name].push(table.name);
|
|
126
|
+
spinner.start('Processing...');
|
|
127
|
+
if (utils.isFiltered(table.name)) {
|
|
128
|
+
spinner.info('Skipped');
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (utils.isCached(table.name, table.content)) {
|
|
132
|
+
spinner.info('From Cache');
|
|
133
|
+
logQueryEvent('cache-query', source.type, source.name);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
logQueryEvent('db-query', source.type, source.name, table.name);
|
|
137
|
+
|
|
138
|
+
if ('url' in table) {
|
|
139
|
+
outputManifest.renderedFiles[source.name].push(table.url);
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
try {
|
|
143
|
+
if (!table.rows) {
|
|
144
|
+
spinner.warn('No results returned.');
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
const outDir = path.join(dataPath, source.name, table.name);
|
|
148
|
+
await fs.mkdir(outDir, { recursive: true });
|
|
149
|
+
const tmpDir = path.join(metaPath, source.name, table.name, 'tmp');
|
|
150
|
+
await fs.mkdir(tmpDir, { recursive: true });
|
|
151
|
+
const filename = table.name + '.parquet';
|
|
152
|
+
const writtenRows = await buildMultipartParquet(
|
|
153
|
+
table.columnTypes,
|
|
154
|
+
table.rows,
|
|
155
|
+
tmpDir,
|
|
156
|
+
outDir,
|
|
157
|
+
filename,
|
|
158
|
+
source.buildOptions.batchSize
|
|
159
|
+
);
|
|
160
|
+
|
|
161
|
+
if (writtenRows === false) {
|
|
162
|
+
throw new EvidenceError('Internal Error', [
|
|
163
|
+
'No rows were written to the filesystem, but the table claimed to contain rows'
|
|
164
|
+
]);
|
|
165
|
+
}
|
|
166
|
+
outputManifest.renderedFiles[source.name].push(
|
|
167
|
+
`${dataUrlPrefix}/${source.name}/${table.name}/${filename}`
|
|
168
|
+
);
|
|
169
|
+
await fs.writeFile(
|
|
170
|
+
path.join(outDir, table.name + '.schema.json'),
|
|
171
|
+
JSON.stringify(table.columnTypes)
|
|
172
|
+
);
|
|
173
|
+
|
|
174
|
+
addToCache(source.name, table.name, table.content);
|
|
175
|
+
|
|
176
|
+
spinner.succeed(`Finished, wrote ${writtenRows} rows.`);
|
|
177
|
+
} catch (e) {
|
|
178
|
+
if (e instanceof Error) {
|
|
179
|
+
console.error(e.stack);
|
|
180
|
+
spinner.fail(e.message);
|
|
181
|
+
} else {
|
|
182
|
+
spinner.fail('Unknown Error Encountered');
|
|
183
|
+
}
|
|
184
|
+
if (e instanceof EvidenceError && e.context) {
|
|
185
|
+
if (Array.isArray(e.context)) console.warn(chalk.dim(' ' + e.context.join('\n ')));
|
|
186
|
+
else console.warn(chalk.dim(' ' + e.context));
|
|
187
|
+
}
|
|
188
|
+
console.debug();
|
|
189
|
+
if (e instanceof Error)
|
|
190
|
+
console.debug(chalk.dim(' ' + e.stack?.split('\n').join('\n ')));
|
|
191
|
+
else console.debug(chalk.dim(e));
|
|
192
|
+
|
|
193
|
+
if (strict) {
|
|
194
|
+
throw e;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
console.log(chalk.dim('-'.repeat(5)));
|
|
200
|
+
|
|
201
|
+
if (skippedSources.length)
|
|
202
|
+
console.log(
|
|
203
|
+
chalk.dim(
|
|
204
|
+
` ${skippedSources.length} source${skippedSources.length === 1 ? '' : 's'} were not run due to filters`
|
|
205
|
+
)
|
|
206
|
+
);
|
|
207
|
+
|
|
208
|
+
await flushCache(metaPath);
|
|
209
|
+
|
|
210
|
+
return outputManifest;
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
*
|
|
215
|
+
* @param {Awaited<ReturnType<typeof loadSources>>[number]} source
|
|
216
|
+
* @param {import('./types.js').SourceFilters} [filters]
|
|
217
|
+
* @returns {import('./types.js').SourceUtils}
|
|
218
|
+
*/
|
|
219
|
+
const buildUtils = (source, filters) => {
|
|
220
|
+
/** @type {import('./types.js').SourceUtils} */
|
|
221
|
+
const utils = {
|
|
222
|
+
/**
|
|
223
|
+
* @param {string} name
|
|
224
|
+
* @param {string} content
|
|
225
|
+
*/
|
|
226
|
+
isCached: (name, content) =>
|
|
227
|
+
Boolean(filters?.only_changed && checkCache(source.name, name, content)),
|
|
228
|
+
/**
|
|
229
|
+
* @param {string} name
|
|
230
|
+
* @returns {boolean} true if query is included in filters
|
|
231
|
+
*/
|
|
232
|
+
isFiltered: (name) => {
|
|
233
|
+
if (!filters?.queries?.size) return false;
|
|
234
|
+
return Boolean(!filters.queries.has(name) && !filters.queries.has(`${source.name}.${name}`));
|
|
235
|
+
},
|
|
236
|
+
/**
|
|
237
|
+
* @param {string} name
|
|
238
|
+
* @param {string} content
|
|
239
|
+
* @returns {boolean}
|
|
240
|
+
*/
|
|
241
|
+
shouldRun: (name, content) => !utils.isFiltered(name) && !utils.isCached(name, content),
|
|
242
|
+
/**
|
|
243
|
+
* @param {string} name
|
|
244
|
+
* @param {string} content
|
|
245
|
+
*/
|
|
246
|
+
addToCache: (name, content) => addToCache(source.name, name, content),
|
|
247
|
+
subSourceVariables: subSourceVariables,
|
|
248
|
+
escape: (tableName, tableContent) => ({
|
|
249
|
+
name: tableName,
|
|
250
|
+
content: tableContent,
|
|
251
|
+
rows: [],
|
|
252
|
+
columnTypes: []
|
|
253
|
+
})
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
return utils;
|
|
257
|
+
};
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { PluginPackageSchema, isDatasourcePlugin } from '../schemas/plugin-package.schema.js';
|
|
3
|
+
|
|
4
|
+
const REGISTRY_URL = 'https://registry.npmjs.com';
|
|
5
|
+
|
|
6
|
+
// These schemas are NOT complete; they only have fields that we care about
|
|
7
|
+
const responseShape = z.object({
|
|
8
|
+
objects: z.array(
|
|
9
|
+
z.object({
|
|
10
|
+
package: z.object({
|
|
11
|
+
name: z.string(),
|
|
12
|
+
description: z.string().optional(),
|
|
13
|
+
version: z.string()
|
|
14
|
+
})
|
|
15
|
+
})
|
|
16
|
+
)
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @returns {Promise<import("./Datasources.js").DatasourcePackage[]>}
|
|
21
|
+
*/
|
|
22
|
+
export const getPublishedDatasources = async () => {
|
|
23
|
+
const searchUrl = REGISTRY_URL + '/-/v1/search?text=keywords:evidence-datasource';
|
|
24
|
+
const response = await fetch(searchUrl).then((r) => r.json());
|
|
25
|
+
|
|
26
|
+
const result = responseShape.parse(response);
|
|
27
|
+
|
|
28
|
+
const output = await Promise.all(
|
|
29
|
+
result.objects.map(async ({ package: pack }) => {
|
|
30
|
+
const metaUrl = REGISTRY_URL + `/${pack.name}`;
|
|
31
|
+
const metaResponse = await fetch(metaUrl).then((r) => r.json());
|
|
32
|
+
if (!('versions' in metaResponse) || typeof metaResponse.versions !== 'object') {
|
|
33
|
+
console.debug(`Malformed response for potential plugin ${pack.name} (missing versions)`);
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
if (!(pack.version in metaResponse.versions)) {
|
|
37
|
+
console.debug(
|
|
38
|
+
`Malformed response for potential plugin ${pack.name} (missing version ${pack.version})`
|
|
39
|
+
);
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
const meta = PluginPackageSchema.parse(metaResponse.versions[pack.version]);
|
|
43
|
+
|
|
44
|
+
if (!isDatasourcePlugin(meta)) {
|
|
45
|
+
console.debug(
|
|
46
|
+
`${pack.name} has evidence-datasource keyword, but is not a valid datasource`
|
|
47
|
+
);
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return meta;
|
|
52
|
+
})
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
return output.filter(
|
|
56
|
+
/**
|
|
57
|
+
* Type narrowing to assert that the values are not null
|
|
58
|
+
* @returns {o is import("./Datasources.js").DatasourcePackage}
|
|
59
|
+
*/
|
|
60
|
+
(o) => o !== null
|
|
61
|
+
);
|
|
62
|
+
};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export * from './schemas/index.js';
|
|
2
|
+
export * from './loadSources.js';
|
|
3
|
+
export * from './loadSourcePlugins.js';
|
|
4
|
+
export * from './writeSourceConfig.js';
|
|
5
|
+
export * from './cli/edit/Options.js';
|
|
6
|
+
export { getDatasourceConfigAsEnvironmentVariables } from './loadSourceConfig.js';
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import chalk from 'chalk';
|
|
4
|
+
import yaml from 'yaml';
|
|
5
|
+
import { redPipe } from '../../lib/tokens.js';
|
|
6
|
+
import { EvidenceError } from '../../lib/EvidenceError.js';
|
|
7
|
+
import { cleanZodErrors } from '../../lib/cleanZodErrors.js';
|
|
8
|
+
import { DatasourceSpecFileSchema } from './schemas/datasource.schema.js';
|
|
9
|
+
import { decodeBase64Deep } from '../../lib/b64Deep.js';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* @param {string} sourceDir
|
|
13
|
+
* @deprecated Use loadSourceConfig
|
|
14
|
+
*/
|
|
15
|
+
export const loadConnectionOptions = async (sourceDir) => {
|
|
16
|
+
const optionsFilePath = path.join(sourceDir, 'connection.options.yaml');
|
|
17
|
+
const optionsFileExists = await fs
|
|
18
|
+
.stat(optionsFilePath)
|
|
19
|
+
.then(() => true)
|
|
20
|
+
.catch(() => false);
|
|
21
|
+
if (!optionsFileExists) return {};
|
|
22
|
+
const optionsFile = await fs.readFile(optionsFilePath).then((r) => r.toString());
|
|
23
|
+
try {
|
|
24
|
+
// TODO: Should we make the b64 encoding optional? (e.g. prefix with b64: to indicate encoding intent)
|
|
25
|
+
return decodeBase64Deep(yaml.parse(optionsFile));
|
|
26
|
+
} catch (e) {
|
|
27
|
+
throw new EvidenceError(`Error parsing connection.options.yaml file; ${sourceDir}`, [], {
|
|
28
|
+
cause: e
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* @param {string} sourceName
|
|
34
|
+
* @deprecated Use loadSourceConfig
|
|
35
|
+
*/
|
|
36
|
+
export const loadConnectionEnvironment = async (sourceName) => {
|
|
37
|
+
/** @type {any} */
|
|
38
|
+
const out = {};
|
|
39
|
+
const keyRegex = /^EVIDENCE_SOURCE__([a-zA-Z0-9_]+)$/;
|
|
40
|
+
for (const [key, value] of Object.entries(process.env)) {
|
|
41
|
+
const parts = keyRegex.exec(key);
|
|
42
|
+
if (!parts) continue;
|
|
43
|
+
if (parts?.length < 2) continue;
|
|
44
|
+
if (!parts[1].toLowerCase().startsWith(sourceName.toLowerCase())) continue;
|
|
45
|
+
const rawOptKey = parts[1].substring(sourceName.length + 2).split('__');
|
|
46
|
+
let t = out;
|
|
47
|
+
|
|
48
|
+
rawOptKey.forEach((key, i) => {
|
|
49
|
+
if (i < rawOptKey.length - 1) {
|
|
50
|
+
// We haven't reached the final key
|
|
51
|
+
if (!t[key]) t[key] = {};
|
|
52
|
+
t = t[key];
|
|
53
|
+
} else {
|
|
54
|
+
t[key] = value;
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* @param {string} sourceDir
|
|
63
|
+
* @deprecated Use loadSourceConfig
|
|
64
|
+
* @returns {Promise<import('./schemas/datasource.schema.js').DatasourceSpec | false>}
|
|
65
|
+
*/
|
|
66
|
+
export const loadConnection = async (sourceDir) => {
|
|
67
|
+
const connParamsRaw = await fs
|
|
68
|
+
.readFile(path.join(sourceDir, 'connection.yaml'))
|
|
69
|
+
.then((r) => r.toString())
|
|
70
|
+
.catch(
|
|
71
|
+
/** @returns {false} */
|
|
72
|
+
(e) => {
|
|
73
|
+
console.warn(
|
|
74
|
+
chalk.yellow(`[!] ${sourceDir} is not a valid source (no connection.yaml); skipping`)
|
|
75
|
+
);
|
|
76
|
+
console.warn(e.message);
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
);
|
|
80
|
+
if (connParamsRaw === false) return false;
|
|
81
|
+
|
|
82
|
+
let connParamsUnchecked;
|
|
83
|
+
try {
|
|
84
|
+
connParamsUnchecked = yaml.parse(connParamsRaw);
|
|
85
|
+
} catch (e) {
|
|
86
|
+
throw new EvidenceError(`Error parsing connection.yaml file; ${sourceDir}`, undefined, {
|
|
87
|
+
cause: e
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const validationResult = DatasourceSpecFileSchema.safeParse(connParamsUnchecked);
|
|
92
|
+
if (!validationResult.success) {
|
|
93
|
+
const formattedError = cleanZodErrors(validationResult.error.format());
|
|
94
|
+
const context = [
|
|
95
|
+
chalk.bold.red(`[!] connection.yaml has errors (${sourceDir}`),
|
|
96
|
+
chalk.red('| Discovered Errors:'),
|
|
97
|
+
`${redPipe} ${yaml.stringify(formattedError).replace(/\n/g, `\n${redPipe} `)}`
|
|
98
|
+
];
|
|
99
|
+
throw new EvidenceError('Unable to load connection.yaml', context);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return validationResult.data;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* @param {string} sourceDir
|
|
107
|
+
* @returns {Promise<import('./schemas/datasource.schema.js').DatasourceSpec | false>}
|
|
108
|
+
*/
|
|
109
|
+
export const loadSourceConfig = async (sourceDir) => {
|
|
110
|
+
const connectionConfig = await loadConnection(sourceDir);
|
|
111
|
+
if (!connectionConfig) return false;
|
|
112
|
+
const options = Object.assign(
|
|
113
|
+
{},
|
|
114
|
+
connectionConfig.options ?? {},
|
|
115
|
+
await loadConnectionOptions(sourceDir),
|
|
116
|
+
await loadConnectionEnvironment(connectionConfig.name)
|
|
117
|
+
);
|
|
118
|
+
return {
|
|
119
|
+
...connectionConfig,
|
|
120
|
+
options
|
|
121
|
+
};
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* @param {import('./schemas/datasource.schema.js').DatasourceSpec} datasource
|
|
126
|
+
*/
|
|
127
|
+
export const getDatasourceConfigAsEnvironmentVariables = (datasource) => {
|
|
128
|
+
/** @type {Record<string,string>} */
|
|
129
|
+
const environmentVariables = {};
|
|
130
|
+
/**
|
|
131
|
+
* @param {any} obj
|
|
132
|
+
* @param {string} [currentKey]
|
|
133
|
+
*/
|
|
134
|
+
const generateNestedEnvVars = (obj, currentKey = '') => {
|
|
135
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
136
|
+
const newKey = currentKey ? `${currentKey}__${key}` : key;
|
|
137
|
+
if (typeof value === 'object') {
|
|
138
|
+
generateNestedEnvVars(value, newKey);
|
|
139
|
+
} else {
|
|
140
|
+
environmentVariables[`EVIDENCE_SOURCE__${datasource.name}__${newKey}`] = value.toString();
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
generateNestedEnvVars(datasource.options);
|
|
145
|
+
|
|
146
|
+
return environmentVariables;
|
|
147
|
+
};
|