@yolean/evidence-sdk 0.0.0-stage → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/CHANGELOG.md +544 -0
  2. package/LICENSE.md +9 -0
  3. package/README.md +237 -2
  4. package/package.json +161 -3
  5. package/src/build-dev/cli/add/addServerHook.js +30 -0
  6. package/src/build-dev/cli/add/addTypeRef.js +19 -0
  7. package/src/build-dev/cli/add/updateGitIgnore.js +33 -0
  8. package/src/build-dev/cli/add/updatePackageJson.js +26 -0
  9. package/src/build-dev/cli/add/updateSvelteConfig.js +199 -0
  10. package/src/build-dev/cli/add/updateViteConfig.js +152 -0
  11. package/src/build-dev/cli/add.js +95 -0
  12. package/src/build-dev/cli/build.js +30 -0
  13. package/src/build-dev/cli/dev.js +49 -0
  14. package/src/build-dev/cli/index.js +14 -0
  15. package/src/build-dev/cli/upgrade.js +54 -0
  16. package/src/build-dev/svelte/index.js +54 -0
  17. package/src/build-dev/svelte/lib/isTargetFile.js +6 -0
  18. package/src/build-dev/svelte/metadatas.js +39 -0
  19. package/src/build-dev/svelte/processors/addBasePathToHrefAndSrc/addBasePathToHrefAndSrc.js +29 -0
  20. package/src/build-dev/svelte/processors/addBasePathToHrefAndSrc/addBasePathToHrefAndSrc.spec.js +82 -0
  21. package/src/build-dev/svelte/processors/addScriptTags.js +17 -0
  22. package/src/build-dev/svelte/processors/convertToHtml/convertToHtml.js +19 -0
  23. package/src/build-dev/svelte/processors/convertToHtml/convertToHtml.spec.js +123 -0
  24. package/src/build-dev/svelte/processors/convertToHtml/processMarkdown.js +59 -0
  25. package/src/build-dev/svelte/processors/extractQueries/extractQueries.js +49 -0
  26. package/src/build-dev/svelte/processors/extractQueries/extractQueries.spec.js +54 -0
  27. package/src/build-dev/svelte/processors/highlight/highlight.js +52 -0
  28. package/src/build-dev/svelte/processors/highlight/highlight.spec.js +31 -0
  29. package/src/build-dev/svelte/processors/initializeUniversalSql.js +15 -0
  30. package/src/build-dev/svelte/processors/injectComponents/injectComponents.js +56 -0
  31. package/src/build-dev/svelte/processors/injectHighlightStyles.js +14 -0
  32. package/src/build-dev/svelte/processors/injectQueries/injectQueries.js +77 -0
  33. package/src/build-dev/svelte/processors/markRootLayoutPrerenderable.js +56 -0
  34. package/src/build-dev/svelte/processors/obfuscateCode/obfuscateCode.js +17 -0
  35. package/src/build-dev/svelte/processors/obfuscateCode/obfuscateCode.spec.js +21 -0
  36. package/src/build-dev/svelte/processors/transformQueries/transformQueries.js +58 -0
  37. package/src/build-dev/vite/config-virtual.js +24 -0
  38. package/src/build-dev/vite/constants.js +7 -0
  39. package/src/build-dev/vite/datasource-plugins/source-query-hmr.js +117 -0
  40. package/src/build-dev/vite/evidence-config.d.ts +6 -0
  41. package/src/build-dev/vite/index.js +214 -0
  42. package/src/build-dev/vite/layout-plugins/copy-to-layout.js +71 -0
  43. package/src/build-dev/vite/plugins.js +4 -0
  44. package/src/build-dev/vite/query-directory-hmr.js +32 -0
  45. package/src/build-dev/vite/sidecar-app.js +79 -0
  46. package/src/build-dev/vite/virtuals/QuerySSR.svelte +13 -0
  47. package/src/build-dev/vite/virtuals/__tests/proxyStack.spec.js +52 -0
  48. package/src/build-dev/vite/virtuals/bootstrapContexts.svelte.js +27 -0
  49. package/src/build-dev/vite/virtuals/browser/static-assets.js +3 -0
  50. package/src/build-dev/vite/virtuals/contextKeys.js +3 -0
  51. package/src/build-dev/vite/virtuals/initUsql.js +23 -0
  52. package/src/build-dev/vite/virtuals/node/static-assets.js +56 -0
  53. package/src/build-dev/vite/virtuals/proxyStack.js +37 -0
  54. package/src/build-dev/vite/virtuals/queries.svelte.js +150 -0
  55. package/src/build-dev/vite/virtuals/ssrHook.svelte.js +60 -0
  56. package/src/build-dev/vite/virtuals/updateQueryContext.svelte.js +31 -0
  57. package/src/build-dev/vite/virtuals-dynamic.js +12 -0
  58. package/src/cli.js +5 -0
  59. package/src/cli.root.js +90 -0
  60. package/src/configuration/cli/create.js +20 -0
  61. package/src/configuration/cli/index.js +12 -0
  62. package/src/configuration/createEvidenceConfig.js +51 -0
  63. package/src/configuration/getEvidenceConfig.js +60 -0
  64. package/src/configuration/getEvidenceConfig.legacy.js +27 -0
  65. package/src/configuration/index.js +1 -0
  66. package/src/configuration/schemas/config.schema.js +11 -0
  67. package/src/configuration/schemas/deployment.schema.js +18 -0
  68. package/src/configuration/schemas/deployment.schema.spec.js +34 -0
  69. package/src/configuration/updateEvidenceConfig.js +25 -0
  70. package/src/legacy-compat.js +8 -0
  71. package/src/lib/EvidenceError.js +35 -0
  72. package/src/lib/b64Deep.js +55 -0
  73. package/src/lib/check-import.js +12 -0
  74. package/src/lib/cleanZodErrors.js +22 -0
  75. package/src/lib/cli/cmdFail.js +33 -0
  76. package/src/lib/cli/notices.js +44 -0
  77. package/src/lib/debug.js +40 -0
  78. package/src/lib/flows.d.ts +21 -0
  79. package/src/lib/forceLink.js +28 -0
  80. package/src/lib/history/index.js +70 -0
  81. package/src/lib/history/types.d.ts +8 -0
  82. package/src/lib/pkg-types.helpers.js +16 -0
  83. package/src/lib/processing-queue.js +96 -0
  84. package/src/lib/processing-queue.spec.js +148 -0
  85. package/src/lib/projectPaths.js +29 -0
  86. package/src/lib/projectRoot.js +12 -0
  87. package/src/lib/sharedPromise.js +77 -0
  88. package/src/lib/sharedPromise.spec.js +23 -0
  89. package/src/lib/strict.js +19 -0
  90. package/src/lib/tests/fs.js +21 -0
  91. package/src/lib/tokens.js +7 -0
  92. package/src/lib/unnest-zod-error.js +14 -0
  93. package/src/logger/EvidenceLogger.js +164 -0
  94. package/src/logger/index.js +4 -0
  95. package/src/meta.js +8 -0
  96. package/src/plugins/cli/index.js +13 -0
  97. package/src/plugins/cli/install.js +26 -0
  98. package/src/plugins/cli/install.prompt.js +121 -0
  99. package/src/plugins/components/getAllComponents.js +82 -0
  100. package/src/plugins/components/getComponentsInPlugin.js +33 -0
  101. package/src/plugins/components/loadComponentPlugins.js +48 -0
  102. package/src/plugins/components/loaders/file-loader.js +125 -0
  103. package/src/plugins/components/loaders/manifest-loader.js +53 -0
  104. package/src/plugins/components/schemas/component-manifest.schema.js +5 -0
  105. package/src/plugins/components/types.d.ts +21 -0
  106. package/src/plugins/components/validateOverrides.js +30 -0
  107. package/src/plugins/components/validateOverrides.spec.js +32 -0
  108. package/src/plugins/datasources/Datasources.js +85 -0
  109. package/src/plugins/datasources/Datasources.spec.js +78 -0
  110. package/src/plugins/datasources/SourceResultCache.js +62 -0
  111. package/src/plugins/datasources/buildSourceDirectoryProxy.js +21 -0
  112. package/src/plugins/datasources/cli/edit/Options.fixtures.js +69 -0
  113. package/src/plugins/datasources/cli/edit/Options.js +253 -0
  114. package/src/plugins/datasources/cli/edit/Options.spec.js +496 -0
  115. package/src/plugins/datasources/cli/edit/editSourceConfig.js +201 -0
  116. package/src/plugins/datasources/cli/edit.js +20 -0
  117. package/src/plugins/datasources/cli/edit.prompt.js +100 -0
  118. package/src/plugins/datasources/cli/index.js +18 -0
  119. package/src/plugins/datasources/cli/sources.js +77 -0
  120. package/src/plugins/datasources/evalSources.js +257 -0
  121. package/src/plugins/datasources/getPublishedDatasources.js +62 -0
  122. package/src/plugins/datasources/index.js +6 -0
  123. package/src/plugins/datasources/loadSourceConfig.js +147 -0
  124. package/src/plugins/datasources/loadSourceConfig.spec.js +127 -0
  125. package/src/plugins/datasources/loadSourcePlugins.js +49 -0
  126. package/src/plugins/datasources/loadSources.js +48 -0
  127. package/src/plugins/datasources/schemas/datasource.schema.js +52 -0
  128. package/src/plugins/datasources/schemas/datasourceCache.schema.js +3 -0
  129. package/src/plugins/datasources/schemas/datasourcePlugin.schema.js +37 -0
  130. package/src/plugins/datasources/schemas/datasourcePluginOptions.schema.js +64 -0
  131. package/src/plugins/datasources/schemas/index.js +2 -0
  132. package/src/plugins/datasources/schemas/manifest.schema.js +8 -0
  133. package/src/plugins/datasources/schemas/results.schema.js +126 -0
  134. package/src/plugins/datasources/sub-source-vars.js +55 -0
  135. package/src/plugins/datasources/sub-source-vars.spec.js +46 -0
  136. package/src/plugins/datasources/types.d.ts +53 -0
  137. package/src/plugins/datasources/updateManifest.js +115 -0
  138. package/src/plugins/datasources/updateManifest.spec.js +200 -0
  139. package/src/plugins/datasources/wrapSimpleConnector.js +95 -0
  140. package/src/plugins/datasources/writeSourceConfig.js +60 -0
  141. package/src/plugins/index.js +2 -0
  142. package/src/plugins/layouts/copyMethods/index.js +10 -0
  143. package/src/plugins/layouts/copyMethods/svelte.js +53 -0
  144. package/src/plugins/layouts/copyMethods/sveltekit.js +75 -0
  145. package/src/plugins/layouts/copyToLayout.js +106 -0
  146. package/src/plugins/layouts/loadLayoutPlugin.js +23 -0
  147. package/src/plugins/loadPluginPackage.js +77 -0
  148. package/src/plugins/schemas/index.js +1 -0
  149. package/src/plugins/schemas/plugin-config.schema.js +41 -0
  150. package/src/plugins/schemas/plugin-config.schema.spec.js +21 -0
  151. package/src/plugins/schemas/plugin-package.schema.js +154 -0
  152. package/src/plugins/schemas/plugin-package.schema.spec.js +30 -0
  153. package/src/query-builder.d.ts +1 -0
  154. package/src/query-builder.js +1 -0
  155. package/src/types/duckdb-wellknown.d.ts +35 -0
  156. package/src/types/html-attributes.ts +13 -0
  157. package/src/types/index.d.ts +18 -0
  158. package/src/types/mosaic-sql.d.ts +27 -0
  159. package/src/types/sveltekit-autoimport.d.ts +13 -0
  160. package/src/types/virtuals.d.ts +113 -0
  161. package/src/types.d.ts +1 -0
  162. package/src/usql/index.d.ts +4 -0
  163. package/src/usql/index.js +3 -0
  164. package/src/usql/query/Query.js +1391 -0
  165. package/src/usql/query/Query.spec.js +827 -0
  166. package/src/usql/query/queryScore.js +92 -0
  167. package/src/usql/query/queryScore.spec.js +26 -0
  168. package/src/usql/query/sterilizeQuery.js +89 -0
  169. package/src/usql/query/sterilizeQuery.spec.js +172 -0
  170. package/src/usql/setTrackProxy/setTrackProxy.js +80 -0
  171. package/src/usql/setTrackProxy/setTrackProxy.spec.js +166 -0
  172. package/src/usql/types.ts +86 -0
  173. package/src/usql/utilities/duckdbSerialize.js +29 -0
  174. package/src/usql/utilities/index.js +2 -0
  175. package/src/usql/utilities/resolveMaybePromise.js +30 -0
  176. package/src/utils/batchUp.js +22 -0
  177. package/src/utils/batchUp.spec.js +69 -0
  178. package/src/utils/browserDebounce.js +16 -0
  179. package/src/utils/index.js +7 -0
  180. package/src/utils/svelte/addBasePath.js +74 -0
  181. package/src/utils/svelte/addBasePath.spec.js +108 -0
  182. package/src/utils/svelte/index.js +13 -0
  183. package/src/utils/svelte/inputs.js +138 -0
  184. package/tsconfig.json +22 -0
  185. package/vite.config.js +3 -0
  186. package/vitest.config.js +15 -0
@@ -0,0 +1,100 @@
1
+ import path from 'path';
2
+ import yaml from 'yaml';
3
+ import fs from 'fs/promises';
4
+
5
+ import { loadSourcePlugins } from '../loadSourcePlugins.js';
6
+ import { loadSources } from '../loadSources.js';
7
+ import * as prompt from '@clack/prompts';
8
+ import { EvidenceError } from '../../../lib/EvidenceError.js';
9
+ import { editSourceConfig } from './edit/editSourceConfig.js';
10
+
11
+ export const editConnectionsPrompt = async () => {
12
+ const dataSpinner = prompt.spinner();
13
+
14
+ dataSpinner.start('Fetching connections and plugins');
15
+ const [sources, plugins] = await Promise.all([loadSources(), loadSourcePlugins()]);
16
+ dataSpinner.stop('Done!', 0);
17
+
18
+ const selectedSourceName = await prompt.select({
19
+ message: 'Which source are we editing?',
20
+ options: [
21
+ ...sources.map((source) => ({
22
+ value: source.name,
23
+ label: `${source.name} (${source.type})`
24
+ })),
25
+ {
26
+ value: null,
27
+ label: 'Create new connection'
28
+ }
29
+ ]
30
+ });
31
+
32
+ if (selectedSourceName) {
33
+ const selectedSource = sources.find((source) => source.name === selectedSourceName);
34
+ if (!selectedSource) throw new EvidenceError('');
35
+ const plugin = plugins.getBySource(selectedSource.type);
36
+ if (!plugin)
37
+ throw new EvidenceError('Tried to edit source that does not have a correlated plugin');
38
+ await editSourceConfig(selectedSource, plugin[1]);
39
+ } else {
40
+ const canonSources = plugins.getCanonicalSources();
41
+
42
+ if (!canonSources.length) {
43
+ prompt.log.error(
44
+ 'No source plugins are installed, or none were found in evidence.config.yaml'
45
+ );
46
+ prompt.cancel('Operation Failed');
47
+
48
+ return false;
49
+ }
50
+
51
+ const selectedType = await prompt.select({
52
+ message: 'What source should we connect to?',
53
+ options: canonSources.map((s) => ({ value: s }))
54
+ });
55
+
56
+ if (prompt.isCancel(selectedType)) {
57
+ prompt.cancel('Did not create connection!');
58
+ return false;
59
+ }
60
+
61
+ /** @type {string | symbol} */
62
+ const name = await prompt
63
+ .text({
64
+ message: 'What should we call this connection?',
65
+ validate: (s) => {
66
+ if (s.length === 0) return 'Connection name must not be empty';
67
+ if (!/^[\w_]+$/.test(s))
68
+ return 'Connection name can only contain letters, numbers, and underscores';
69
+ }
70
+ })
71
+ .then((r) => (prompt.isCancel(r) ? r : r?.toString()));
72
+
73
+ if (prompt.isCancel(name)) {
74
+ prompt.cancel('Did not create connection!');
75
+ return false;
76
+ }
77
+
78
+ const existingSourceDirs = await fs.readdir(path.resolve('sources'));
79
+ if (existingSourceDirs.includes(name))
80
+ throw new EvidenceError(
81
+ `Cannot create source ${name}, directory ./sources/${name} already exists`
82
+ );
83
+ const sourceWithSameName = sources.find((source) => source.name === name);
84
+ if (sourceWithSameName)
85
+ throw new EvidenceError(
86
+ `Cannot create source ${name}, a source with that name already exists in ${sourceWithSameName.dir}`
87
+ );
88
+ await fs.mkdir(path.join(path.resolve('sources'), name));
89
+ const comment = `# This file was automatically generated\n`;
90
+ await fs.writeFile(
91
+ path.join(path.resolve('sources'), name, 'connection.yaml'),
92
+ comment + yaml.stringify({ name, type: selectedType })
93
+ );
94
+
95
+ const newSource = (await loadSources()).find((s) => s.name === name);
96
+ if (!newSource)
97
+ throw new EvidenceError('Internal Error', ['Failed to load just-created source']);
98
+ await editSourceConfig(newSource, plugins.getBySource(newSource.type)[1]);
99
+ }
100
+ };
@@ -0,0 +1,18 @@
1
+ import { sources } from './sources.js';
2
+ import { edit } from './edit.js';
3
+
4
+ /**
5
+ * @type {import("@brianmd/citty").SubCommandsDef}
6
+ */
7
+ export const datasourcesCli = {
8
+ sources,
9
+ connections: {
10
+ meta: {
11
+ name: 'connections',
12
+ description: 'Manage your source connections'
13
+ },
14
+ subCommands: {
15
+ edit
16
+ }
17
+ }
18
+ };
@@ -0,0 +1,77 @@
1
+ import chalk from 'chalk';
2
+ import { dataDirectory, metaDirectory } from '../../../lib/projectPaths.js';
3
+ import { evalSources } from '../evalSources.js';
4
+ import { updateManifest } from '../updateManifest.js';
5
+ import { log } from '../../../logger/index.js';
6
+ import { enableStrictMode, isStrictMode } from '../../../lib/strict.js';
7
+ import { EvidenceError } from '../../../lib/EvidenceError.js';
8
+
9
+ /** @type { import("@brianmd/citty").CommandDef } */
10
+ export const sources = {
11
+ meta: {
12
+ name: 'sources',
13
+ description: 'Update your Evidence source outputs'
14
+ },
15
+
16
+ args: {
17
+ changed: {
18
+ type: 'boolean',
19
+ description: 'Run only queries that have changed',
20
+ default: false
21
+ },
22
+ sources: {
23
+ type: 'string',
24
+ description: "Run only the listed sources (',' delimited)"
25
+ },
26
+ queries: {
27
+ type: 'string',
28
+ description: "Run only the listed queries (',' delimited)"
29
+ },
30
+ strict: {
31
+ type: 'boolean',
32
+ description: 'Fail when a source query fails',
33
+ default: false
34
+ }
35
+ },
36
+
37
+ async run({ args }) {
38
+ if (args.strict) {
39
+ enableStrictMode();
40
+ if (!isStrictMode()) {
41
+ throw new EvidenceError('Failed to enable strict mode');
42
+ }
43
+ }
44
+
45
+ const sources = new Set(
46
+ args.sources
47
+ ?.toString()
48
+ .split(',')
49
+ .map(/** @param {string} s */ (s) => s.trim())
50
+ );
51
+ if (sources.size) log.debug(`\tRunning sources: ${[...sources].join(', ')}`);
52
+
53
+ const queries = new Set(
54
+ args.queries
55
+ ?.toString()
56
+ .split(',')
57
+ .map(/** @param {string} s */ (s) => s.trim())
58
+ );
59
+ if (queries.size) log.debug(`\tRunning queries: ${[...queries].join(', ')}`);
60
+
61
+ const evaluatedManifest = await evalSources(
62
+ dataDirectory,
63
+ metaDirectory,
64
+ {
65
+ sources,
66
+ queries,
67
+ only_changed: args.changed === true
68
+ },
69
+ isStrictMode()
70
+ );
71
+
72
+ log.info(' Evaluated sources, saving manifest');
73
+
74
+ await updateManifest(evaluatedManifest, dataDirectory);
75
+ log.info(chalk.bold.green(' ✅ Done!'));
76
+ }
77
+ };
@@ -0,0 +1,257 @@
1
+ import chalk from 'chalk';
2
+ import fs from 'fs/promises';
3
+ import path from 'path';
4
+
5
+ import { EvidenceError } from '../../lib/EvidenceError.js';
6
+ import { loadSourcePlugins } from './loadSourcePlugins.js';
7
+ import { loadSources } from './loadSources.js';
8
+ import { wrapSimpleConnector } from './wrapSimpleConnector.js';
9
+ import { buildMultipartParquet } from '@evidence-dev/universal-sql';
10
+ import { buildSourceDirectoryProxy } from './buildSourceDirectoryProxy.js';
11
+ import { addToCache, checkCache, flushCache, loadCache } from './SourceResultCache.js';
12
+ import ora from 'ora';
13
+ import { dataUrlPrefix } from '../../lib/projectPaths.js';
14
+ import { subSourceVariables } from './sub-source-vars.js';
15
+ import { logQueryEvent } from '@evidence-dev/telemetry';
16
+
17
+ // TODO: This is a great candidate for unit testing - but it may need to be broken down further to make that more achievable, right now it would take a _lot_ of mocks
18
+
19
+ /**
20
+ * @param {string} dataPath
21
+ * @param {string} metaPath
22
+ * @param {import('./types.js').SourceFilters} [filters] `sources` or `queries` being null means no filter
23
+ * @param {boolean} [strict]
24
+ * @returns {Promise<import('./types.js').Manifest>}
25
+ */
26
+ export const evalSources = async (dataPath, metaPath, filters, strict) => {
27
+ const pluginLoader = ora({ text: 'Loading plugins & sources' }).start();
28
+
29
+ // Setup work
30
+ const [sourcePlugins, sources] = await Promise.all([
31
+ loadSourcePlugins(),
32
+ loadSources(pluginLoader),
33
+ loadCache(metaPath)
34
+ ]).catch((e) => {
35
+ pluginLoader.fail();
36
+ throw e;
37
+ });
38
+
39
+ if (sources.length) {
40
+ pluginLoader.succeed();
41
+ }
42
+
43
+ /** @type {import('./types.js').Manifest} */
44
+ const outputManifest = { renderedFiles: {}, locatedFiles: {}, locatedSchemas: [] };
45
+
46
+ /** @type {string[]} */
47
+ const skippedSources = [];
48
+
49
+ for (const source of sources) {
50
+ outputManifest.locatedSchemas ??= [];
51
+ outputManifest.locatedSchemas.push(source.name);
52
+ if (filters?.sources?.size && !filters?.sources?.has(source.name)) {
53
+ console.debug(` [Skipping]: ${chalk.bold(source.name)}`);
54
+ skippedSources.push(source.name);
55
+ continue;
56
+ } else {
57
+ console.log(chalk.dim('-'.repeat(5)));
58
+ console.log(chalk.green(` [Processing] ${chalk.bold(source.name)}`));
59
+ }
60
+
61
+ const plugin = sourcePlugins.getBySource(source.type);
62
+ if (!plugin) {
63
+ logQueryEvent('source-connector-not-found', source.type, source.name);
64
+ // TODO: How forgiving do we want to be here?
65
+ // TODO: If we want to be really fancy; we could batch these, and do an NPM lookup at the end to say "these packages provide those datasources"
66
+ throw new EvidenceError(
67
+ `Could not find matching datasource plugin for ${chalk.bold(
68
+ source.name
69
+ )} (source: ${chalk.bold(source.type)})`
70
+ );
71
+ }
72
+ const [, mod] = plugin;
73
+ const testResult = await mod.testConnection(source.options, source.dir);
74
+ if (testResult !== true) {
75
+ logQueryEvent('db-connection-error', source.type, source.name);
76
+ throw new EvidenceError(
77
+ `Error connecting to datasource ${chalk.bold(source.name)}: ${testResult.reason}`
78
+ );
79
+ } else {
80
+ logQueryEvent('db-connection-success', source.type, source.name);
81
+ }
82
+
83
+ /** @type {import('./types.js').ProcessSourceFn} */
84
+ const tableIter = 'processSource' in mod ? mod.processSource : wrapSimpleConnector(mod, source);
85
+
86
+ const utils = buildUtils(source, filters);
87
+
88
+ outputManifest.renderedFiles[source.name] = [];
89
+ if (!outputManifest.locatedFiles) outputManifest.locatedFiles = {};
90
+ outputManifest.locatedFiles[source.name] = [];
91
+
92
+ const iter = tableIter(source.options, await buildSourceDirectoryProxy(source.dir), utils)[
93
+ Symbol.asyncIterator
94
+ ](); // this is required for typescript to be happy
95
+
96
+ /** @type {IteratorResult<import('./types.js').QueryResultTable<any> | EvidenceError>} */
97
+ let iterResult;
98
+
99
+ while (((iterResult = await iter.next()), !iterResult.done)) {
100
+ if (iterResult.done) continue;
101
+ if (iterResult.value instanceof Error) {
102
+ const error = iterResult.value;
103
+ let tableName = 'Unknown';
104
+ if (error instanceof EvidenceError) {
105
+ if (error.metadata.tableName) tableName = error.metadata.tableName;
106
+ }
107
+ ora({
108
+ prefixText: ` ${tableName}`,
109
+ spinner: 'triangle',
110
+ discardStdin: false,
111
+ interval: 250
112
+ }).fail(`Error: ${error.message}`);
113
+ logQueryEvent('db-error', source.type, source.name, tableName);
114
+ if (strict) throw error;
115
+ continue;
116
+ }
117
+ const table = { ...iterResult.value };
118
+ const spinner = ora({
119
+ prefixText: ` ${table.name}`,
120
+ spinner: 'triangle',
121
+ discardStdin: false,
122
+ interval: 250
123
+ });
124
+
125
+ outputManifest.locatedFiles[source.name].push(table.name);
126
+ spinner.start('Processing...');
127
+ if (utils.isFiltered(table.name)) {
128
+ spinner.info('Skipped');
129
+ continue;
130
+ }
131
+ if (utils.isCached(table.name, table.content)) {
132
+ spinner.info('From Cache');
133
+ logQueryEvent('cache-query', source.type, source.name);
134
+ continue;
135
+ }
136
+ logQueryEvent('db-query', source.type, source.name, table.name);
137
+
138
+ if ('url' in table) {
139
+ outputManifest.renderedFiles[source.name].push(table.url);
140
+ continue;
141
+ }
142
+ try {
143
+ if (!table.rows) {
144
+ spinner.warn('No results returned.');
145
+ continue;
146
+ }
147
+ const outDir = path.join(dataPath, source.name, table.name);
148
+ await fs.mkdir(outDir, { recursive: true });
149
+ const tmpDir = path.join(metaPath, source.name, table.name, 'tmp');
150
+ await fs.mkdir(tmpDir, { recursive: true });
151
+ const filename = table.name + '.parquet';
152
+ const writtenRows = await buildMultipartParquet(
153
+ table.columnTypes,
154
+ table.rows,
155
+ tmpDir,
156
+ outDir,
157
+ filename,
158
+ source.buildOptions.batchSize
159
+ );
160
+
161
+ if (writtenRows === false) {
162
+ throw new EvidenceError('Internal Error', [
163
+ 'No rows were written to the filesystem, but the table claimed to contain rows'
164
+ ]);
165
+ }
166
+ outputManifest.renderedFiles[source.name].push(
167
+ `${dataUrlPrefix}/${source.name}/${table.name}/${filename}`
168
+ );
169
+ await fs.writeFile(
170
+ path.join(outDir, table.name + '.schema.json'),
171
+ JSON.stringify(table.columnTypes)
172
+ );
173
+
174
+ addToCache(source.name, table.name, table.content);
175
+
176
+ spinner.succeed(`Finished, wrote ${writtenRows} rows.`);
177
+ } catch (e) {
178
+ if (e instanceof Error) {
179
+ console.error(e.stack);
180
+ spinner.fail(e.message);
181
+ } else {
182
+ spinner.fail('Unknown Error Encountered');
183
+ }
184
+ if (e instanceof EvidenceError && e.context) {
185
+ if (Array.isArray(e.context)) console.warn(chalk.dim(' ' + e.context.join('\n ')));
186
+ else console.warn(chalk.dim(' ' + e.context));
187
+ }
188
+ console.debug();
189
+ if (e instanceof Error)
190
+ console.debug(chalk.dim(' ' + e.stack?.split('\n').join('\n ')));
191
+ else console.debug(chalk.dim(e));
192
+
193
+ if (strict) {
194
+ throw e;
195
+ }
196
+ }
197
+ }
198
+ }
199
+ console.log(chalk.dim('-'.repeat(5)));
200
+
201
+ if (skippedSources.length)
202
+ console.log(
203
+ chalk.dim(
204
+ ` ${skippedSources.length} source${skippedSources.length === 1 ? '' : 's'} were not run due to filters`
205
+ )
206
+ );
207
+
208
+ await flushCache(metaPath);
209
+
210
+ return outputManifest;
211
+ };
212
+
213
+ /**
214
+ *
215
+ * @param {Awaited<ReturnType<typeof loadSources>>[number]} source
216
+ * @param {import('./types.js').SourceFilters} [filters]
217
+ * @returns {import('./types.js').SourceUtils}
218
+ */
219
+ const buildUtils = (source, filters) => {
220
+ /** @type {import('./types.js').SourceUtils} */
221
+ const utils = {
222
+ /**
223
+ * @param {string} name
224
+ * @param {string} content
225
+ */
226
+ isCached: (name, content) =>
227
+ Boolean(filters?.only_changed && checkCache(source.name, name, content)),
228
+ /**
229
+ * @param {string} name
230
+ * @returns {boolean} true if query is included in filters
231
+ */
232
+ isFiltered: (name) => {
233
+ if (!filters?.queries?.size) return false;
234
+ return Boolean(!filters.queries.has(name) && !filters.queries.has(`${source.name}.${name}`));
235
+ },
236
+ /**
237
+ * @param {string} name
238
+ * @param {string} content
239
+ * @returns {boolean}
240
+ */
241
+ shouldRun: (name, content) => !utils.isFiltered(name) && !utils.isCached(name, content),
242
+ /**
243
+ * @param {string} name
244
+ * @param {string} content
245
+ */
246
+ addToCache: (name, content) => addToCache(source.name, name, content),
247
+ subSourceVariables: subSourceVariables,
248
+ escape: (tableName, tableContent) => ({
249
+ name: tableName,
250
+ content: tableContent,
251
+ rows: [],
252
+ columnTypes: []
253
+ })
254
+ };
255
+
256
+ return utils;
257
+ };
@@ -0,0 +1,62 @@
1
+ import { z } from 'zod';
2
+ import { PluginPackageSchema, isDatasourcePlugin } from '../schemas/plugin-package.schema.js';
3
+
4
+ const REGISTRY_URL = 'https://registry.npmjs.com';
5
+
6
+ // These schemas are NOT complete; they only have fields that we care about
7
+ const responseShape = z.object({
8
+ objects: z.array(
9
+ z.object({
10
+ package: z.object({
11
+ name: z.string(),
12
+ description: z.string().optional(),
13
+ version: z.string()
14
+ })
15
+ })
16
+ )
17
+ });
18
+
19
+ /**
20
+ * @returns {Promise<import("./Datasources.js").DatasourcePackage[]>}
21
+ */
22
+ export const getPublishedDatasources = async () => {
23
+ const searchUrl = REGISTRY_URL + '/-/v1/search?text=keywords:evidence-datasource';
24
+ const response = await fetch(searchUrl).then((r) => r.json());
25
+
26
+ const result = responseShape.parse(response);
27
+
28
+ const output = await Promise.all(
29
+ result.objects.map(async ({ package: pack }) => {
30
+ const metaUrl = REGISTRY_URL + `/${pack.name}`;
31
+ const metaResponse = await fetch(metaUrl).then((r) => r.json());
32
+ if (!('versions' in metaResponse) || typeof metaResponse.versions !== 'object') {
33
+ console.debug(`Malformed response for potential plugin ${pack.name} (missing versions)`);
34
+ return null;
35
+ }
36
+ if (!(pack.version in metaResponse.versions)) {
37
+ console.debug(
38
+ `Malformed response for potential plugin ${pack.name} (missing version ${pack.version})`
39
+ );
40
+ return null;
41
+ }
42
+ const meta = PluginPackageSchema.parse(metaResponse.versions[pack.version]);
43
+
44
+ if (!isDatasourcePlugin(meta)) {
45
+ console.debug(
46
+ `${pack.name} has evidence-datasource keyword, but is not a valid datasource`
47
+ );
48
+ return null;
49
+ }
50
+
51
+ return meta;
52
+ })
53
+ );
54
+
55
+ return output.filter(
56
+ /**
57
+ * Type narrowing to assert that the values are not null
58
+ * @returns {o is import("./Datasources.js").DatasourcePackage}
59
+ */
60
+ (o) => o !== null
61
+ );
62
+ };
@@ -0,0 +1,6 @@
1
+ export * from './schemas/index.js';
2
+ export * from './loadSources.js';
3
+ export * from './loadSourcePlugins.js';
4
+ export * from './writeSourceConfig.js';
5
+ export * from './cli/edit/Options.js';
6
+ export { getDatasourceConfigAsEnvironmentVariables } from './loadSourceConfig.js';
@@ -0,0 +1,147 @@
1
+ import fs from 'fs/promises';
2
+ import path from 'path';
3
+ import chalk from 'chalk';
4
+ import yaml from 'yaml';
5
+ import { redPipe } from '../../lib/tokens.js';
6
+ import { EvidenceError } from '../../lib/EvidenceError.js';
7
+ import { cleanZodErrors } from '../../lib/cleanZodErrors.js';
8
+ import { DatasourceSpecFileSchema } from './schemas/datasource.schema.js';
9
+ import { decodeBase64Deep } from '../../lib/b64Deep.js';
10
+
11
+ /**
12
+ * @param {string} sourceDir
13
+ * @deprecated Use loadSourceConfig
14
+ */
15
+ export const loadConnectionOptions = async (sourceDir) => {
16
+ const optionsFilePath = path.join(sourceDir, 'connection.options.yaml');
17
+ const optionsFileExists = await fs
18
+ .stat(optionsFilePath)
19
+ .then(() => true)
20
+ .catch(() => false);
21
+ if (!optionsFileExists) return {};
22
+ const optionsFile = await fs.readFile(optionsFilePath).then((r) => r.toString());
23
+ try {
24
+ // TODO: Should we make the b64 encoding optional? (e.g. prefix with b64: to indicate encoding intent)
25
+ return decodeBase64Deep(yaml.parse(optionsFile));
26
+ } catch (e) {
27
+ throw new EvidenceError(`Error parsing connection.options.yaml file; ${sourceDir}`, [], {
28
+ cause: e
29
+ });
30
+ }
31
+ };
32
+ /**
33
+ * @param {string} sourceName
34
+ * @deprecated Use loadSourceConfig
35
+ */
36
+ export const loadConnectionEnvironment = async (sourceName) => {
37
+ /** @type {any} */
38
+ const out = {};
39
+ const keyRegex = /^EVIDENCE_SOURCE__([a-zA-Z0-9_]+)$/;
40
+ for (const [key, value] of Object.entries(process.env)) {
41
+ const parts = keyRegex.exec(key);
42
+ if (!parts) continue;
43
+ if (parts?.length < 2) continue;
44
+ if (!parts[1].toLowerCase().startsWith(sourceName.toLowerCase())) continue;
45
+ const rawOptKey = parts[1].substring(sourceName.length + 2).split('__');
46
+ let t = out;
47
+
48
+ rawOptKey.forEach((key, i) => {
49
+ if (i < rawOptKey.length - 1) {
50
+ // We haven't reached the final key
51
+ if (!t[key]) t[key] = {};
52
+ t = t[key];
53
+ } else {
54
+ t[key] = value;
55
+ }
56
+ });
57
+ }
58
+ return out;
59
+ };
60
+
61
+ /**
62
+ * @param {string} sourceDir
63
+ * @deprecated Use loadSourceConfig
64
+ * @returns {Promise<import('./schemas/datasource.schema.js').DatasourceSpec | false>}
65
+ */
66
+ export const loadConnection = async (sourceDir) => {
67
+ const connParamsRaw = await fs
68
+ .readFile(path.join(sourceDir, 'connection.yaml'))
69
+ .then((r) => r.toString())
70
+ .catch(
71
+ /** @returns {false} */
72
+ (e) => {
73
+ console.warn(
74
+ chalk.yellow(`[!] ${sourceDir} is not a valid source (no connection.yaml); skipping`)
75
+ );
76
+ console.warn(e.message);
77
+ return false;
78
+ }
79
+ );
80
+ if (connParamsRaw === false) return false;
81
+
82
+ let connParamsUnchecked;
83
+ try {
84
+ connParamsUnchecked = yaml.parse(connParamsRaw);
85
+ } catch (e) {
86
+ throw new EvidenceError(`Error parsing connection.yaml file; ${sourceDir}`, undefined, {
87
+ cause: e
88
+ });
89
+ }
90
+
91
+ const validationResult = DatasourceSpecFileSchema.safeParse(connParamsUnchecked);
92
+ if (!validationResult.success) {
93
+ const formattedError = cleanZodErrors(validationResult.error.format());
94
+ const context = [
95
+ chalk.bold.red(`[!] connection.yaml has errors (${sourceDir}`),
96
+ chalk.red('| Discovered Errors:'),
97
+ `${redPipe} ${yaml.stringify(formattedError).replace(/\n/g, `\n${redPipe} `)}`
98
+ ];
99
+ throw new EvidenceError('Unable to load connection.yaml', context);
100
+ }
101
+
102
+ return validationResult.data;
103
+ };
104
+
105
+ /**
106
+ * @param {string} sourceDir
107
+ * @returns {Promise<import('./schemas/datasource.schema.js').DatasourceSpec | false>}
108
+ */
109
+ export const loadSourceConfig = async (sourceDir) => {
110
+ const connectionConfig = await loadConnection(sourceDir);
111
+ if (!connectionConfig) return false;
112
+ const options = Object.assign(
113
+ {},
114
+ connectionConfig.options ?? {},
115
+ await loadConnectionOptions(sourceDir),
116
+ await loadConnectionEnvironment(connectionConfig.name)
117
+ );
118
+ return {
119
+ ...connectionConfig,
120
+ options
121
+ };
122
+ };
123
+
124
+ /**
125
+ * @param {import('./schemas/datasource.schema.js').DatasourceSpec} datasource
126
+ */
127
+ export const getDatasourceConfigAsEnvironmentVariables = (datasource) => {
128
+ /** @type {Record<string,string>} */
129
+ const environmentVariables = {};
130
+ /**
131
+ * @param {any} obj
132
+ * @param {string} [currentKey]
133
+ */
134
+ const generateNestedEnvVars = (obj, currentKey = '') => {
135
+ for (const [key, value] of Object.entries(obj)) {
136
+ const newKey = currentKey ? `${currentKey}__${key}` : key;
137
+ if (typeof value === 'object') {
138
+ generateNestedEnvVars(value, newKey);
139
+ } else {
140
+ environmentVariables[`EVIDENCE_SOURCE__${datasource.name}__${newKey}`] = value.toString();
141
+ }
142
+ }
143
+ };
144
+ generateNestedEnvVars(datasource.options);
145
+
146
+ return environmentVariables;
147
+ };