@yolean/evidence-sdk 0.0.0-stage → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +544 -0
- package/LICENSE.md +9 -0
- package/README.md +237 -2
- package/package.json +161 -3
- package/src/build-dev/cli/add/addServerHook.js +30 -0
- package/src/build-dev/cli/add/addTypeRef.js +19 -0
- package/src/build-dev/cli/add/updateGitIgnore.js +33 -0
- package/src/build-dev/cli/add/updatePackageJson.js +26 -0
- package/src/build-dev/cli/add/updateSvelteConfig.js +199 -0
- package/src/build-dev/cli/add/updateViteConfig.js +152 -0
- package/src/build-dev/cli/add.js +95 -0
- package/src/build-dev/cli/build.js +30 -0
- package/src/build-dev/cli/dev.js +49 -0
- package/src/build-dev/cli/index.js +14 -0
- package/src/build-dev/cli/upgrade.js +54 -0
- package/src/build-dev/svelte/index.js +54 -0
- package/src/build-dev/svelte/lib/isTargetFile.js +6 -0
- package/src/build-dev/svelte/metadatas.js +39 -0
- package/src/build-dev/svelte/processors/addBasePathToHrefAndSrc/addBasePathToHrefAndSrc.js +29 -0
- package/src/build-dev/svelte/processors/addBasePathToHrefAndSrc/addBasePathToHrefAndSrc.spec.js +82 -0
- package/src/build-dev/svelte/processors/addScriptTags.js +17 -0
- package/src/build-dev/svelte/processors/convertToHtml/convertToHtml.js +19 -0
- package/src/build-dev/svelte/processors/convertToHtml/convertToHtml.spec.js +123 -0
- package/src/build-dev/svelte/processors/convertToHtml/processMarkdown.js +59 -0
- package/src/build-dev/svelte/processors/extractQueries/extractQueries.js +49 -0
- package/src/build-dev/svelte/processors/extractQueries/extractQueries.spec.js +54 -0
- package/src/build-dev/svelte/processors/highlight/highlight.js +52 -0
- package/src/build-dev/svelte/processors/highlight/highlight.spec.js +31 -0
- package/src/build-dev/svelte/processors/initializeUniversalSql.js +15 -0
- package/src/build-dev/svelte/processors/injectComponents/injectComponents.js +56 -0
- package/src/build-dev/svelte/processors/injectHighlightStyles.js +14 -0
- package/src/build-dev/svelte/processors/injectQueries/injectQueries.js +77 -0
- package/src/build-dev/svelte/processors/markRootLayoutPrerenderable.js +56 -0
- package/src/build-dev/svelte/processors/obfuscateCode/obfuscateCode.js +17 -0
- package/src/build-dev/svelte/processors/obfuscateCode/obfuscateCode.spec.js +21 -0
- package/src/build-dev/svelte/processors/transformQueries/transformQueries.js +58 -0
- package/src/build-dev/vite/config-virtual.js +24 -0
- package/src/build-dev/vite/constants.js +7 -0
- package/src/build-dev/vite/datasource-plugins/source-query-hmr.js +117 -0
- package/src/build-dev/vite/evidence-config.d.ts +6 -0
- package/src/build-dev/vite/index.js +214 -0
- package/src/build-dev/vite/layout-plugins/copy-to-layout.js +71 -0
- package/src/build-dev/vite/plugins.js +4 -0
- package/src/build-dev/vite/query-directory-hmr.js +32 -0
- package/src/build-dev/vite/sidecar-app.js +79 -0
- package/src/build-dev/vite/virtuals/QuerySSR.svelte +13 -0
- package/src/build-dev/vite/virtuals/__tests/proxyStack.spec.js +52 -0
- package/src/build-dev/vite/virtuals/bootstrapContexts.svelte.js +27 -0
- package/src/build-dev/vite/virtuals/browser/static-assets.js +3 -0
- package/src/build-dev/vite/virtuals/contextKeys.js +3 -0
- package/src/build-dev/vite/virtuals/initUsql.js +23 -0
- package/src/build-dev/vite/virtuals/node/static-assets.js +56 -0
- package/src/build-dev/vite/virtuals/proxyStack.js +37 -0
- package/src/build-dev/vite/virtuals/queries.svelte.js +150 -0
- package/src/build-dev/vite/virtuals/ssrHook.svelte.js +60 -0
- package/src/build-dev/vite/virtuals/updateQueryContext.svelte.js +31 -0
- package/src/build-dev/vite/virtuals-dynamic.js +12 -0
- package/src/cli.js +5 -0
- package/src/cli.root.js +90 -0
- package/src/configuration/cli/create.js +20 -0
- package/src/configuration/cli/index.js +12 -0
- package/src/configuration/createEvidenceConfig.js +51 -0
- package/src/configuration/getEvidenceConfig.js +60 -0
- package/src/configuration/getEvidenceConfig.legacy.js +27 -0
- package/src/configuration/index.js +1 -0
- package/src/configuration/schemas/config.schema.js +11 -0
- package/src/configuration/schemas/deployment.schema.js +18 -0
- package/src/configuration/schemas/deployment.schema.spec.js +34 -0
- package/src/configuration/updateEvidenceConfig.js +25 -0
- package/src/legacy-compat.js +8 -0
- package/src/lib/EvidenceError.js +35 -0
- package/src/lib/b64Deep.js +55 -0
- package/src/lib/check-import.js +12 -0
- package/src/lib/cleanZodErrors.js +22 -0
- package/src/lib/cli/cmdFail.js +33 -0
- package/src/lib/cli/notices.js +44 -0
- package/src/lib/debug.js +40 -0
- package/src/lib/flows.d.ts +21 -0
- package/src/lib/forceLink.js +28 -0
- package/src/lib/history/index.js +70 -0
- package/src/lib/history/types.d.ts +8 -0
- package/src/lib/pkg-types.helpers.js +16 -0
- package/src/lib/processing-queue.js +96 -0
- package/src/lib/processing-queue.spec.js +148 -0
- package/src/lib/projectPaths.js +29 -0
- package/src/lib/projectRoot.js +12 -0
- package/src/lib/sharedPromise.js +77 -0
- package/src/lib/sharedPromise.spec.js +23 -0
- package/src/lib/strict.js +19 -0
- package/src/lib/tests/fs.js +21 -0
- package/src/lib/tokens.js +7 -0
- package/src/lib/unnest-zod-error.js +14 -0
- package/src/logger/EvidenceLogger.js +164 -0
- package/src/logger/index.js +4 -0
- package/src/meta.js +8 -0
- package/src/plugins/cli/index.js +13 -0
- package/src/plugins/cli/install.js +26 -0
- package/src/plugins/cli/install.prompt.js +121 -0
- package/src/plugins/components/getAllComponents.js +82 -0
- package/src/plugins/components/getComponentsInPlugin.js +33 -0
- package/src/plugins/components/loadComponentPlugins.js +48 -0
- package/src/plugins/components/loaders/file-loader.js +125 -0
- package/src/plugins/components/loaders/manifest-loader.js +53 -0
- package/src/plugins/components/schemas/component-manifest.schema.js +5 -0
- package/src/plugins/components/types.d.ts +21 -0
- package/src/plugins/components/validateOverrides.js +30 -0
- package/src/plugins/components/validateOverrides.spec.js +32 -0
- package/src/plugins/datasources/Datasources.js +85 -0
- package/src/plugins/datasources/Datasources.spec.js +78 -0
- package/src/plugins/datasources/SourceResultCache.js +62 -0
- package/src/plugins/datasources/buildSourceDirectoryProxy.js +21 -0
- package/src/plugins/datasources/cli/edit/Options.fixtures.js +69 -0
- package/src/plugins/datasources/cli/edit/Options.js +253 -0
- package/src/plugins/datasources/cli/edit/Options.spec.js +496 -0
- package/src/plugins/datasources/cli/edit/editSourceConfig.js +201 -0
- package/src/plugins/datasources/cli/edit.js +20 -0
- package/src/plugins/datasources/cli/edit.prompt.js +100 -0
- package/src/plugins/datasources/cli/index.js +18 -0
- package/src/plugins/datasources/cli/sources.js +77 -0
- package/src/plugins/datasources/evalSources.js +257 -0
- package/src/plugins/datasources/getPublishedDatasources.js +62 -0
- package/src/plugins/datasources/index.js +6 -0
- package/src/plugins/datasources/loadSourceConfig.js +147 -0
- package/src/plugins/datasources/loadSourceConfig.spec.js +127 -0
- package/src/plugins/datasources/loadSourcePlugins.js +49 -0
- package/src/plugins/datasources/loadSources.js +48 -0
- package/src/plugins/datasources/schemas/datasource.schema.js +52 -0
- package/src/plugins/datasources/schemas/datasourceCache.schema.js +3 -0
- package/src/plugins/datasources/schemas/datasourcePlugin.schema.js +37 -0
- package/src/plugins/datasources/schemas/datasourcePluginOptions.schema.js +64 -0
- package/src/plugins/datasources/schemas/index.js +2 -0
- package/src/plugins/datasources/schemas/manifest.schema.js +8 -0
- package/src/plugins/datasources/schemas/results.schema.js +126 -0
- package/src/plugins/datasources/sub-source-vars.js +55 -0
- package/src/plugins/datasources/sub-source-vars.spec.js +46 -0
- package/src/plugins/datasources/types.d.ts +53 -0
- package/src/plugins/datasources/updateManifest.js +115 -0
- package/src/plugins/datasources/updateManifest.spec.js +200 -0
- package/src/plugins/datasources/wrapSimpleConnector.js +95 -0
- package/src/plugins/datasources/writeSourceConfig.js +60 -0
- package/src/plugins/index.js +2 -0
- package/src/plugins/layouts/copyMethods/index.js +10 -0
- package/src/plugins/layouts/copyMethods/svelte.js +53 -0
- package/src/plugins/layouts/copyMethods/sveltekit.js +75 -0
- package/src/plugins/layouts/copyToLayout.js +106 -0
- package/src/plugins/layouts/loadLayoutPlugin.js +23 -0
- package/src/plugins/loadPluginPackage.js +77 -0
- package/src/plugins/schemas/index.js +1 -0
- package/src/plugins/schemas/plugin-config.schema.js +41 -0
- package/src/plugins/schemas/plugin-config.schema.spec.js +21 -0
- package/src/plugins/schemas/plugin-package.schema.js +154 -0
- package/src/plugins/schemas/plugin-package.schema.spec.js +30 -0
- package/src/query-builder.d.ts +1 -0
- package/src/query-builder.js +1 -0
- package/src/types/duckdb-wellknown.d.ts +35 -0
- package/src/types/html-attributes.ts +13 -0
- package/src/types/index.d.ts +18 -0
- package/src/types/mosaic-sql.d.ts +27 -0
- package/src/types/sveltekit-autoimport.d.ts +13 -0
- package/src/types/virtuals.d.ts +113 -0
- package/src/types.d.ts +1 -0
- package/src/usql/index.d.ts +4 -0
- package/src/usql/index.js +3 -0
- package/src/usql/query/Query.js +1391 -0
- package/src/usql/query/Query.spec.js +827 -0
- package/src/usql/query/queryScore.js +92 -0
- package/src/usql/query/queryScore.spec.js +26 -0
- package/src/usql/query/sterilizeQuery.js +89 -0
- package/src/usql/query/sterilizeQuery.spec.js +172 -0
- package/src/usql/setTrackProxy/setTrackProxy.js +80 -0
- package/src/usql/setTrackProxy/setTrackProxy.spec.js +166 -0
- package/src/usql/types.ts +86 -0
- package/src/usql/utilities/duckdbSerialize.js +29 -0
- package/src/usql/utilities/index.js +2 -0
- package/src/usql/utilities/resolveMaybePromise.js +30 -0
- package/src/utils/batchUp.js +22 -0
- package/src/utils/batchUp.spec.js +69 -0
- package/src/utils/browserDebounce.js +16 -0
- package/src/utils/index.js +7 -0
- package/src/utils/svelte/addBasePath.js +74 -0
- package/src/utils/svelte/addBasePath.spec.js +108 -0
- package/src/utils/svelte/index.js +13 -0
- package/src/utils/svelte/inputs.js +138 -0
- package/tsconfig.json +22 -0
- package/vite.config.js +3 -0
- package/vitest.config.js +15 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { ManifestSchema } from './schemas/manifest.schema.js';
|
|
4
|
+
import merge from 'lodash.merge';
|
|
5
|
+
import chalk from 'chalk';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
*
|
|
9
|
+
* @param {string} dataDir
|
|
10
|
+
* @returns
|
|
11
|
+
*/
|
|
12
|
+
const safeLoadManifest = async (dataDir) => {
|
|
13
|
+
const manifestPath = path.join(dataDir, 'manifest.json');
|
|
14
|
+
const dirContent = await fs.readdir(dataDir);
|
|
15
|
+
if (!dirContent.includes('manifest.json')) return null;
|
|
16
|
+
if (!(await fs.stat(manifestPath)).isFile()) return null;
|
|
17
|
+
const m = await fs.readFile(manifestPath, 'utf-8');
|
|
18
|
+
try {
|
|
19
|
+
const parsed = JSON.parse(m);
|
|
20
|
+
const valid = ManifestSchema.parse(parsed);
|
|
21
|
+
return valid;
|
|
22
|
+
} catch {
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* @param {import('./types.js').Manifest} updatedManifest
|
|
29
|
+
* @param {string} dataDir
|
|
30
|
+
* @returns {Promise<import('./types.js').Manifest>}
|
|
31
|
+
*/
|
|
32
|
+
export const updateManifest = async (updatedManifest, dataDir) => {
|
|
33
|
+
await fs.mkdir(path.join(dataDir), { recursive: true });
|
|
34
|
+
|
|
35
|
+
const existingManifest = await safeLoadManifest(dataDir);
|
|
36
|
+
|
|
37
|
+
const finalManifest = merge({}, existingManifest); // deep clone
|
|
38
|
+
|
|
39
|
+
if (!updatedManifest.locatedFiles) {
|
|
40
|
+
console.warn(chalk.yellow('[!] No queries were located, operation was a was no-op'));
|
|
41
|
+
return updatedManifest;
|
|
42
|
+
}
|
|
43
|
+
if (!updatedManifest.locatedSchemas) {
|
|
44
|
+
console.warn(chalk.yellow('[!] No sources were located, operation was a was no-op'));
|
|
45
|
+
return updatedManifest;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (existingManifest) {
|
|
49
|
+
for (const schema of updatedManifest.locatedSchemas) {
|
|
50
|
+
console.debug(" Updating schema '" + schema + "'");
|
|
51
|
+
if (!(schema in existingManifest.renderedFiles)) {
|
|
52
|
+
console.debug(' | Schema is new');
|
|
53
|
+
console.debug(
|
|
54
|
+
` | ${updatedManifest.renderedFiles[schema].length} ${updatedManifest.renderedFiles[schema].length === 1 ? 'query' : 'queries'} will be added`
|
|
55
|
+
);
|
|
56
|
+
finalManifest.renderedFiles[schema] = updatedManifest.renderedFiles[schema];
|
|
57
|
+
continue;
|
|
58
|
+
} else {
|
|
59
|
+
console.debug(' | Schema exists already');
|
|
60
|
+
}
|
|
61
|
+
const locatedQueries = updatedManifest.locatedFiles[schema];
|
|
62
|
+
if (!locatedQueries?.length) {
|
|
63
|
+
console.debug(' | No queries were located, but schema exists already');
|
|
64
|
+
console.debug(' | Previous files will be kept');
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
console.debug(
|
|
68
|
+
` | ${locatedQueries.length} ${locatedQueries.length === 1 ? 'query' : 'queries'} found`
|
|
69
|
+
);
|
|
70
|
+
locatedQueries.forEach((q) => console.debug(' | ' + q));
|
|
71
|
+
const newQueries = updatedManifest.renderedFiles[schema].filter((queryPath) => {
|
|
72
|
+
return !existingManifest.renderedFiles[schema].includes(queryPath);
|
|
73
|
+
});
|
|
74
|
+
console.debug(
|
|
75
|
+
` | ${newQueries.length} ${newQueries.length === 1 ? 'query' : 'queries'} are new`
|
|
76
|
+
);
|
|
77
|
+
newQueries.forEach((q) => console.debug(' | ' + q));
|
|
78
|
+
const existingQueries = existingManifest.renderedFiles[schema]
|
|
79
|
+
.filter((queryPath) => {
|
|
80
|
+
return locatedQueries.includes(path.basename(queryPath, '.parquet'));
|
|
81
|
+
})
|
|
82
|
+
.filter(
|
|
83
|
+
// we want to prefer the new query path
|
|
84
|
+
// file basenames should never conflict, so we can safely use that as a ref
|
|
85
|
+
(queryPath) =>
|
|
86
|
+
!newQueries.some(
|
|
87
|
+
(newQueryPath) =>
|
|
88
|
+
path.basename(newQueryPath, '.parquet') === path.basename(queryPath, '.parquet')
|
|
89
|
+
)
|
|
90
|
+
);
|
|
91
|
+
console.debug(
|
|
92
|
+
` | ${existingQueries.length} ${existingQueries.length === 1 ? 'query' : 'queries'} already ${existingQueries.length ? 'exists' : 'exist'}`
|
|
93
|
+
);
|
|
94
|
+
existingQueries.forEach((q) => console.debug(' | ' + q));
|
|
95
|
+
|
|
96
|
+
const result = [...existingQueries, ...newQueries].sort((a, b) => a.localeCompare(b));
|
|
97
|
+
console.debug(
|
|
98
|
+
` | ${result.length} ${result.length === 1 ? 'query' : 'queries'} to be rendered`
|
|
99
|
+
);
|
|
100
|
+
result.forEach((q) => console.debug(' | ' + q));
|
|
101
|
+
finalManifest.renderedFiles[schema] = result;
|
|
102
|
+
}
|
|
103
|
+
for (const schema of Object.keys(existingManifest.renderedFiles)) {
|
|
104
|
+
if (!updatedManifest.locatedSchemas.includes(schema)) {
|
|
105
|
+
console.debug(" Removing schema '" + schema + "'");
|
|
106
|
+
delete finalManifest.renderedFiles[schema];
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
} else {
|
|
110
|
+
Object.assign(finalManifest, { renderedFiles: updatedManifest.renderedFiles });
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
await fs.writeFile(path.join(dataDir, 'manifest.json'), JSON.stringify(finalManifest));
|
|
114
|
+
return finalManifest;
|
|
115
|
+
};
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { resetFs } from '../../lib/tests/fs.js';
|
|
2
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
3
|
+
import fs from 'fs/promises';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
|
|
6
|
+
import { updateManifest } from './updateManifest.js';
|
|
7
|
+
|
|
8
|
+
describe('updateManifest', () => {
|
|
9
|
+
beforeEach(() => {
|
|
10
|
+
resetFs();
|
|
11
|
+
});
|
|
12
|
+
afterEach(() => {
|
|
13
|
+
resetFs();
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it('should write a manifest to disk when no manifest exists', async () => {
|
|
17
|
+
const manifest = {
|
|
18
|
+
renderedFiles: {
|
|
19
|
+
csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet']
|
|
20
|
+
},
|
|
21
|
+
locatedFiles: {
|
|
22
|
+
csv: ['data', 'nullish']
|
|
23
|
+
},
|
|
24
|
+
locatedSchemas: ['csv']
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const dataDir = '/_evidence';
|
|
28
|
+
await updateManifest(manifest, dataDir);
|
|
29
|
+
|
|
30
|
+
expect(await fs.stat(path.join(dataDir, 'manifest.json'))).toBeDefined();
|
|
31
|
+
expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
|
|
32
|
+
JSON.stringify({ renderedFiles: manifest.renderedFiles })
|
|
33
|
+
);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('should remove queries that were not updated, but also were not located', async () => {
|
|
37
|
+
const fsManifest = {
|
|
38
|
+
renderedFiles: {
|
|
39
|
+
csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet']
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
const updatedManifest = {
|
|
43
|
+
renderedFiles: {
|
|
44
|
+
csv: ['/_evidence/query/csv/data.parquet']
|
|
45
|
+
},
|
|
46
|
+
locatedFiles: {
|
|
47
|
+
csv: ['data']
|
|
48
|
+
},
|
|
49
|
+
locatedSchemas: ['csv']
|
|
50
|
+
};
|
|
51
|
+
const dataDir = '/_evidence';
|
|
52
|
+
await fs.mkdir(path.join(dataDir), { recursive: true });
|
|
53
|
+
await fs.writeFile(
|
|
54
|
+
path.join(dataDir, 'manifest.json'),
|
|
55
|
+
JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
|
|
56
|
+
);
|
|
57
|
+
await updateManifest(updatedManifest, dataDir);
|
|
58
|
+
|
|
59
|
+
expect(await fs.stat(path.join(dataDir, 'manifest.json'))).toBeDefined();
|
|
60
|
+
expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
|
|
61
|
+
JSON.stringify({ renderedFiles: updatedManifest.renderedFiles })
|
|
62
|
+
);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('should not remove queries that were not updated, but were located', async () => {
|
|
66
|
+
const fsManifest = {
|
|
67
|
+
renderedFiles: {
|
|
68
|
+
csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet']
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
const updatedManifest = {
|
|
72
|
+
renderedFiles: fsManifest.renderedFiles,
|
|
73
|
+
locatedFiles: {
|
|
74
|
+
csv: ['data', 'nullish']
|
|
75
|
+
},
|
|
76
|
+
locatedSchemas: ['csv']
|
|
77
|
+
};
|
|
78
|
+
const dataDir = '/_evidence';
|
|
79
|
+
await fs.mkdir(path.join(dataDir), { recursive: true });
|
|
80
|
+
await fs.writeFile(
|
|
81
|
+
path.join(dataDir, 'manifest.json'),
|
|
82
|
+
JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
|
|
83
|
+
);
|
|
84
|
+
await updateManifest(updatedManifest, dataDir);
|
|
85
|
+
|
|
86
|
+
expect(await fs.stat(path.join(dataDir, 'manifest.json'))).toBeDefined();
|
|
87
|
+
expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
|
|
88
|
+
JSON.stringify(fsManifest)
|
|
89
|
+
);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it('should not touch schemas that were not updated', async () => {
|
|
93
|
+
const fsManifest = {
|
|
94
|
+
renderedFiles: {
|
|
95
|
+
csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet'],
|
|
96
|
+
json: ['/_evidence/query/json/data.parquet']
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
const updatedManifest = {
|
|
100
|
+
renderedFiles: {
|
|
101
|
+
csv: ['/_evidence/query/csv/data.parquet']
|
|
102
|
+
},
|
|
103
|
+
locatedFiles: {
|
|
104
|
+
csv: ['data', 'nullish']
|
|
105
|
+
},
|
|
106
|
+
locatedSchemas: ['csv', 'json']
|
|
107
|
+
};
|
|
108
|
+
const dataDir = '/_evidence';
|
|
109
|
+
await fs.mkdir(path.join(dataDir), { recursive: true });
|
|
110
|
+
await fs.writeFile(
|
|
111
|
+
path.join(dataDir, 'manifest.json'),
|
|
112
|
+
JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
|
|
113
|
+
);
|
|
114
|
+
await updateManifest(updatedManifest, dataDir);
|
|
115
|
+
|
|
116
|
+
expect(await fs.stat(path.join(dataDir, 'manifest.json'))).toBeDefined();
|
|
117
|
+
expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
|
|
118
|
+
JSON.stringify(fsManifest)
|
|
119
|
+
);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it('should prefer the updated path for a query', async () => {
|
|
123
|
+
const fsManifest = {
|
|
124
|
+
renderedFiles: {
|
|
125
|
+
csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet']
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
const updatedManifest = {
|
|
129
|
+
renderedFiles: {
|
|
130
|
+
csv: ['/__evidence/query/csv/data.parquet', '/__evidence/query/csv/nullish.parquet']
|
|
131
|
+
},
|
|
132
|
+
locatedFiles: {
|
|
133
|
+
csv: ['data', 'nullish']
|
|
134
|
+
},
|
|
135
|
+
locatedSchemas: ['csv']
|
|
136
|
+
};
|
|
137
|
+
const dataDir = '/_evidence';
|
|
138
|
+
await fs.mkdir(path.join(dataDir), { recursive: true });
|
|
139
|
+
await fs.writeFile(
|
|
140
|
+
path.join(dataDir, 'manifest.json'),
|
|
141
|
+
JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
|
|
142
|
+
);
|
|
143
|
+
await updateManifest(updatedManifest, dataDir);
|
|
144
|
+
expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
|
|
145
|
+
JSON.stringify({
|
|
146
|
+
renderedFiles: updatedManifest.renderedFiles
|
|
147
|
+
})
|
|
148
|
+
);
|
|
149
|
+
});
|
|
150
|
+
it('should merge a new source correctly', async () => {
|
|
151
|
+
const fsManifest = {
|
|
152
|
+
renderedFiles: {
|
|
153
|
+
needful_things: ['static/data/needful_things/orders/orders.parquet']
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
const updatedManifest = {
|
|
157
|
+
renderedFiles: {
|
|
158
|
+
my_csv: ['static/data/my_csv/x/x.parquet'],
|
|
159
|
+
needful_things: ['static/data/needful_things/orders/orders.parquet']
|
|
160
|
+
},
|
|
161
|
+
locatedFiles: {
|
|
162
|
+
my_csv: ['x'],
|
|
163
|
+
needful_things: ['needful_things', 'orders']
|
|
164
|
+
},
|
|
165
|
+
locatedSchemas: ['my_csv', 'needful_things']
|
|
166
|
+
};
|
|
167
|
+
const dataDir = '/_evidence';
|
|
168
|
+
await fs.mkdir(path.join(dataDir), { recursive: true });
|
|
169
|
+
await fs.writeFile(
|
|
170
|
+
path.join(dataDir, 'manifest.json'),
|
|
171
|
+
JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
|
|
172
|
+
);
|
|
173
|
+
await updateManifest(updatedManifest, dataDir);
|
|
174
|
+
expect(JSON.parse(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8'))).toEqual({
|
|
175
|
+
renderedFiles: updatedManifest.renderedFiles
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
it('should remove an old source correctly', async () => {
|
|
179
|
+
const fsManifest = {
|
|
180
|
+
renderedFiles: {
|
|
181
|
+
needful_things: ['static/data/needful_things/orders/orders.parquet']
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
const updatedManifest = {
|
|
185
|
+
renderedFiles: {},
|
|
186
|
+
locatedFiles: {},
|
|
187
|
+
locatedSchemas: []
|
|
188
|
+
};
|
|
189
|
+
const dataDir = '/_evidence';
|
|
190
|
+
await fs.mkdir(path.join(dataDir), { recursive: true });
|
|
191
|
+
await fs.writeFile(
|
|
192
|
+
path.join(dataDir, 'manifest.json'),
|
|
193
|
+
JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
|
|
194
|
+
);
|
|
195
|
+
await updateManifest(updatedManifest, dataDir);
|
|
196
|
+
expect(JSON.parse(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8'))).toEqual({
|
|
197
|
+
renderedFiles: updatedManifest.renderedFiles
|
|
198
|
+
});
|
|
199
|
+
});
|
|
200
|
+
});
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import { EvidenceError } from '../../lib/EvidenceError.js';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import { readFileSync, statSync } from 'fs';
|
|
5
|
+
import chalk from 'chalk';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @param {import("./Datasources.js").Datasource} mod
|
|
9
|
+
* @param {import('./schemas/datasource.schema.js').DatasourceSpec & {dir: string}} source
|
|
10
|
+
* @returns {import('./types.js').ProcessSourceFn}
|
|
11
|
+
*/
|
|
12
|
+
export const wrapSimpleConnector = (mod, source) => {
|
|
13
|
+
if (!('getRunner' in mod))
|
|
14
|
+
throw new EvidenceError(
|
|
15
|
+
'Internal Error',
|
|
16
|
+
'Advanced connector was passed to wrapSimpleConnector. This should not occur'
|
|
17
|
+
);
|
|
18
|
+
/**
|
|
19
|
+
* @type {import('./types.js').ProcessSourceFn}
|
|
20
|
+
*/
|
|
21
|
+
return async function* (...args) {
|
|
22
|
+
const [, , utils] = args;
|
|
23
|
+
const runner = await mod.getRunner(source.options, source.dir);
|
|
24
|
+
// TODO: Use fsproxy instead of fs
|
|
25
|
+
/**
|
|
26
|
+
*
|
|
27
|
+
* @param {string} dir
|
|
28
|
+
* @returns {AsyncGenerator<import('./types.js').QueryResultTable | EvidenceError>}
|
|
29
|
+
*/
|
|
30
|
+
const processDir = async function* (dir) {
|
|
31
|
+
const sourceFiles = await fs.readdir(dir, { withFileTypes: true });
|
|
32
|
+
for (const sourceFile of sourceFiles) {
|
|
33
|
+
// Why is the dirent interface so unstable?
|
|
34
|
+
// This behaves differently depending on which minor version of 18 / 20 you are using
|
|
35
|
+
const dirPath =
|
|
36
|
+
// @ts-expect-error
|
|
37
|
+
'parentPath' in sourceFile ? sourceFile.parentPath + '' : (sourceFile.path ?? dir);
|
|
38
|
+
if (sourceFile.name === 'connection.yaml' || sourceFile.name === 'connection.options.yaml')
|
|
39
|
+
continue;
|
|
40
|
+
if (sourceFile.isDirectory()) {
|
|
41
|
+
yield* processDir(path.join(dirPath, sourceFile.name));
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
if (!sourceFile.isFile()) continue;
|
|
46
|
+
|
|
47
|
+
const sourceFilePath = path.join(dirPath, sourceFile.name);
|
|
48
|
+
const sourceFileName = sourceFile.name.split('.').at(0) ?? '';
|
|
49
|
+
|
|
50
|
+
const stat = statSync(sourceFilePath);
|
|
51
|
+
let sourceFileContent;
|
|
52
|
+
if (stat.size > 1024 * 1024 * 32 /* 32 Megabytes */) {
|
|
53
|
+
console.debug(chalk.dim('Will not eagerly load files larger than 32 Megabytes.'));
|
|
54
|
+
sourceFileContent = '';
|
|
55
|
+
} else {
|
|
56
|
+
sourceFileContent = readFileSync(sourceFilePath, 'utf-8');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Check if it should be skipped, and return an empty result (this is handled above)
|
|
60
|
+
if (!utils.shouldRun(sourceFileName, sourceFileContent)) {
|
|
61
|
+
yield utils.escape(sourceFileName, sourceFileContent);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
try {
|
|
66
|
+
yield {
|
|
67
|
+
name: /** @type {string} */ (sourceFileName),
|
|
68
|
+
content: sourceFileContent,
|
|
69
|
+
columnTypes: [],
|
|
70
|
+
...(await runner(
|
|
71
|
+
utils.subSourceVariables(sourceFileContent),
|
|
72
|
+
sourceFilePath,
|
|
73
|
+
source.buildOptions.batchSize ?? 1000 * 1000 // TODO: BatchSize configurable? Perhaps per-source plugin or per connection
|
|
74
|
+
))
|
|
75
|
+
};
|
|
76
|
+
} catch (e) {
|
|
77
|
+
let message = JSON.stringify(e);
|
|
78
|
+
if (e instanceof Error) message = e.message;
|
|
79
|
+
if (e instanceof EvidenceError) {
|
|
80
|
+
e.metadata.tableName = sourceFileName;
|
|
81
|
+
yield e;
|
|
82
|
+
}
|
|
83
|
+
yield new EvidenceError(
|
|
84
|
+
message,
|
|
85
|
+
[`${source.name}/${sourceFileName ?? ''} failed to execute`],
|
|
86
|
+
{ cause: e },
|
|
87
|
+
{ tableName: sourceFileName }
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
yield* processDir(source.dir);
|
|
94
|
+
};
|
|
95
|
+
};
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import yaml from 'yaml';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import fs from 'fs/promises';
|
|
4
|
+
import { getSafeOptions, getSecretOptions } from './cli/edit/Options.js';
|
|
5
|
+
import { encodeBase64Deep } from '../../lib/b64Deep.js';
|
|
6
|
+
import { sourcesDirectory } from '../../lib/projectPaths.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
*
|
|
10
|
+
* @param {import("./cli/edit/Options.js").OptionsObj} options
|
|
11
|
+
* @param {import("./schemas/datasource.schema.js").DatasourceSpec & { dir?: string }} source
|
|
12
|
+
* @param {import('../../lib/flows.js').EvidenceLoadIndicator} [spinner]
|
|
13
|
+
* @returns {Promise<import("./schemas/datasource.schema.js").DatasourceSpec & { dir: string }>} Source, with the directory attached if it was not previously
|
|
14
|
+
*/
|
|
15
|
+
export const writeSourceConfig = async (options, source, spinner) => {
|
|
16
|
+
const secrets = getSecretOptions(options);
|
|
17
|
+
const safes = getSafeOptions(options);
|
|
18
|
+
|
|
19
|
+
const safesFileString = [
|
|
20
|
+
`# This file was automatically generated`,
|
|
21
|
+
yaml.stringify({ ...source, options: safes, dir: undefined })
|
|
22
|
+
]
|
|
23
|
+
.join('\n')
|
|
24
|
+
.replace(
|
|
25
|
+
'buildOptions:',
|
|
26
|
+
[`# Advanced Options, you probably don't want to change this`, `buildOptions:`].join('\n')
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
const secretsFileString = [
|
|
30
|
+
`# This file was automatically generated`,
|
|
31
|
+
`# Do not edit this file manually; instead use npx evidence-sdk sources edit`,
|
|
32
|
+
``,
|
|
33
|
+
`# It should *not* be source controlled, as it likely contain credentials or other sensitive configuration values.`,
|
|
34
|
+
`# Values in this file are base64 encoded; https://it-tools.tech/base64-string-converter has an excellent encoder / decoder tool.`,
|
|
35
|
+
`# Base64 is NOT encryption, and should not be treated as secure`,
|
|
36
|
+
yaml.stringify(encodeBase64Deep(secrets))
|
|
37
|
+
].join('\n');
|
|
38
|
+
|
|
39
|
+
if (
|
|
40
|
+
!source.dir ||
|
|
41
|
+
!path.dirname(source.dir).startsWith(sourcesDirectory) ||
|
|
42
|
+
!path.isAbsolute(source.dir)
|
|
43
|
+
) {
|
|
44
|
+
source.dir = path.join(sourcesDirectory, source.name);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const safesPath = path.join(source.dir, 'connection.yaml');
|
|
48
|
+
const secretsPath = path.join(source.dir, 'connection.options.yaml');
|
|
49
|
+
|
|
50
|
+
await fs.mkdir(source.dir, { recursive: true });
|
|
51
|
+
|
|
52
|
+
spinner?.start(`Saving ${source.name}`);
|
|
53
|
+
await fs.writeFile(safesPath, safesFileString);
|
|
54
|
+
await fs.writeFile(secretsPath, secretsFileString);
|
|
55
|
+
spinner?.stop(`Finished saving ${source.name}`);
|
|
56
|
+
|
|
57
|
+
return /** @type {import("./schemas/datasource.schema.js").DatasourceSpec & { dir: string }} */ (
|
|
58
|
+
source
|
|
59
|
+
);
|
|
60
|
+
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import * as sveltekit from './sveltekit.js';
|
|
2
|
+
import * as svelte from './svelte.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @type {Record<string, { copyAll: (rootDir: string, rootTargetDir: string) => Promise<void>, copyFile: (rootDir: string, rootTargetDir: string, file: import("fs").Dirent) => void }>}
|
|
6
|
+
*/
|
|
7
|
+
export const copyMethods = {
|
|
8
|
+
sveltekit: sveltekit,
|
|
9
|
+
svelte: svelte
|
|
10
|
+
};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import fs from 'fs/promises';
|
|
3
|
+
import chalk from 'chalk';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
*
|
|
7
|
+
* @param {string} rootDir
|
|
8
|
+
* @param {string} rootTargetDir
|
|
9
|
+
* @param {import("fs").Dirent} file
|
|
10
|
+
*/
|
|
11
|
+
const getTargets = async (rootDir, rootTargetDir, file) => {
|
|
12
|
+
const relativePath = path.relative(path.resolve(rootDir), path.resolve(file.path, file.name));
|
|
13
|
+
const absolutePath = path.join(rootTargetDir, relativePath);
|
|
14
|
+
|
|
15
|
+
return {
|
|
16
|
+
relativePath,
|
|
17
|
+
absolutePath
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* @param {string} rootDir
|
|
23
|
+
* @param {string} rootTargetDir
|
|
24
|
+
* @param {import("fs").Dirent} page
|
|
25
|
+
*/
|
|
26
|
+
export const copyFile = async (rootDir, rootTargetDir, page) => {
|
|
27
|
+
await fs.mkdir(rootTargetDir, { recursive: true });
|
|
28
|
+
const { relativePath } = await getTargets(rootDir, rootTargetDir, page);
|
|
29
|
+
const p = path.parse(path.join(rootTargetDir, relativePath));
|
|
30
|
+
const from = path.join(page.path, page.name);
|
|
31
|
+
let to = path.join(p.dir, page.name);
|
|
32
|
+
await fs.cp(from, to);
|
|
33
|
+
console.debug(chalk.dim(`Copied ${from} -> ${to}`));
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @param {string} rootDir
|
|
38
|
+
* @param {string} rootTargetDir
|
|
39
|
+
*/
|
|
40
|
+
export const copyAll = async (rootDir, rootTargetDir) => {
|
|
41
|
+
await fs.mkdir(rootTargetDir, { recursive: true });
|
|
42
|
+
|
|
43
|
+
const pages = await fs.readdir(path.resolve(rootDir), { recursive: true, withFileTypes: true });
|
|
44
|
+
|
|
45
|
+
for (const page of pages) {
|
|
46
|
+
const { absolutePath } = await getTargets(rootDir, rootTargetDir, page);
|
|
47
|
+
if (page.isDirectory()) {
|
|
48
|
+
await fs.mkdir(absolutePath, { recursive: true });
|
|
49
|
+
} else {
|
|
50
|
+
await copyFile(rootDir, rootTargetDir, page);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import fs from 'fs/promises';
|
|
3
|
+
import chalk from 'chalk';
|
|
4
|
+
import { projectRoot } from '../../../lib/projectRoot.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
*
|
|
8
|
+
* @param {string} rootDir
|
|
9
|
+
* @param {string} rootTargetDir
|
|
10
|
+
* @param {import("fs").Dirent} file
|
|
11
|
+
*/
|
|
12
|
+
const getTargets = async (rootDir, rootTargetDir, file) => {
|
|
13
|
+
const relativePath = path.relative(path.resolve(rootDir), path.resolve(file.path, file.name));
|
|
14
|
+
const absolutePath = path.join(rootTargetDir, relativePath);
|
|
15
|
+
|
|
16
|
+
return {
|
|
17
|
+
relativePath,
|
|
18
|
+
absolutePath
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* @param {string} rootDir
|
|
24
|
+
* @param {string} rootTargetDir
|
|
25
|
+
* @param {import("fs").Dirent} page
|
|
26
|
+
*/
|
|
27
|
+
export const copyFile = async (rootDir, rootTargetDir, page) => {
|
|
28
|
+
// TODO: Error handling
|
|
29
|
+
if (!rootTargetDir.includes('(evidence-pages)'))
|
|
30
|
+
rootTargetDir = path.join(rootTargetDir, '(evidence-pages)');
|
|
31
|
+
await fs.mkdir(rootTargetDir, { recursive: true });
|
|
32
|
+
|
|
33
|
+
const { relativePath } = await getTargets(rootDir, rootTargetDir, page);
|
|
34
|
+
const p = path.parse(path.join(rootTargetDir, relativePath));
|
|
35
|
+
const filename = page.name.split('.')[0]; // rip off the filename
|
|
36
|
+
const extention = page.name.split('.').slice(1).join('.'); // rip off the filename
|
|
37
|
+
const from = path.join(page.path, page.name);
|
|
38
|
+
let to;
|
|
39
|
+
if (page.name === '+layout.svelte') {
|
|
40
|
+
to = path.join(p.dir, page.name);
|
|
41
|
+
} else if (page.name.split('.')[0] === 'index') {
|
|
42
|
+
to = path.join(p.dir, `+page.${extention}`);
|
|
43
|
+
} else {
|
|
44
|
+
// needs a dir
|
|
45
|
+
// same as relative, excludes file extension
|
|
46
|
+
const dirname = path.join(
|
|
47
|
+
rootTargetDir,
|
|
48
|
+
path.relative(path.resolve(projectRoot, 'pages'), path.resolve(page.path, filename))
|
|
49
|
+
);
|
|
50
|
+
await fs.mkdir(dirname, { recursive: true });
|
|
51
|
+
to = path.join(dirname, `+page.${extention}`);
|
|
52
|
+
}
|
|
53
|
+
await fs.cp(from, to);
|
|
54
|
+
console.debug(chalk.dim(`Copied ${from} -> ${to}`));
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* @param {string} rootDir
|
|
59
|
+
* @param {string} rootTargetDir
|
|
60
|
+
*/
|
|
61
|
+
export const copyAll = async (rootDir, rootTargetDir) => {
|
|
62
|
+
rootTargetDir = path.join(rootTargetDir, '(evidence-pages)');
|
|
63
|
+
await fs.mkdir(rootTargetDir, { recursive: true });
|
|
64
|
+
|
|
65
|
+
const pages = await fs.readdir(path.resolve(rootDir), { recursive: true, withFileTypes: true });
|
|
66
|
+
|
|
67
|
+
for (const page of pages) {
|
|
68
|
+
const { absolutePath } = await getTargets(rootDir, rootTargetDir, page);
|
|
69
|
+
if (page.isDirectory()) {
|
|
70
|
+
await fs.mkdir(absolutePath, { recursive: true });
|
|
71
|
+
} else {
|
|
72
|
+
await copyFile(rootDir, rootTargetDir, page);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
};
|