@yolean/evidence-sdk 0.0.0-stage → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/CHANGELOG.md +544 -0
  2. package/LICENSE.md +9 -0
  3. package/README.md +237 -2
  4. package/package.json +161 -3
  5. package/src/build-dev/cli/add/addServerHook.js +30 -0
  6. package/src/build-dev/cli/add/addTypeRef.js +19 -0
  7. package/src/build-dev/cli/add/updateGitIgnore.js +33 -0
  8. package/src/build-dev/cli/add/updatePackageJson.js +26 -0
  9. package/src/build-dev/cli/add/updateSvelteConfig.js +199 -0
  10. package/src/build-dev/cli/add/updateViteConfig.js +152 -0
  11. package/src/build-dev/cli/add.js +95 -0
  12. package/src/build-dev/cli/build.js +30 -0
  13. package/src/build-dev/cli/dev.js +49 -0
  14. package/src/build-dev/cli/index.js +14 -0
  15. package/src/build-dev/cli/upgrade.js +54 -0
  16. package/src/build-dev/svelte/index.js +54 -0
  17. package/src/build-dev/svelte/lib/isTargetFile.js +6 -0
  18. package/src/build-dev/svelte/metadatas.js +39 -0
  19. package/src/build-dev/svelte/processors/addBasePathToHrefAndSrc/addBasePathToHrefAndSrc.js +29 -0
  20. package/src/build-dev/svelte/processors/addBasePathToHrefAndSrc/addBasePathToHrefAndSrc.spec.js +82 -0
  21. package/src/build-dev/svelte/processors/addScriptTags.js +17 -0
  22. package/src/build-dev/svelte/processors/convertToHtml/convertToHtml.js +19 -0
  23. package/src/build-dev/svelte/processors/convertToHtml/convertToHtml.spec.js +123 -0
  24. package/src/build-dev/svelte/processors/convertToHtml/processMarkdown.js +59 -0
  25. package/src/build-dev/svelte/processors/extractQueries/extractQueries.js +49 -0
  26. package/src/build-dev/svelte/processors/extractQueries/extractQueries.spec.js +54 -0
  27. package/src/build-dev/svelte/processors/highlight/highlight.js +52 -0
  28. package/src/build-dev/svelte/processors/highlight/highlight.spec.js +31 -0
  29. package/src/build-dev/svelte/processors/initializeUniversalSql.js +15 -0
  30. package/src/build-dev/svelte/processors/injectComponents/injectComponents.js +56 -0
  31. package/src/build-dev/svelte/processors/injectHighlightStyles.js +14 -0
  32. package/src/build-dev/svelte/processors/injectQueries/injectQueries.js +77 -0
  33. package/src/build-dev/svelte/processors/markRootLayoutPrerenderable.js +56 -0
  34. package/src/build-dev/svelte/processors/obfuscateCode/obfuscateCode.js +17 -0
  35. package/src/build-dev/svelte/processors/obfuscateCode/obfuscateCode.spec.js +21 -0
  36. package/src/build-dev/svelte/processors/transformQueries/transformQueries.js +58 -0
  37. package/src/build-dev/vite/config-virtual.js +24 -0
  38. package/src/build-dev/vite/constants.js +7 -0
  39. package/src/build-dev/vite/datasource-plugins/source-query-hmr.js +117 -0
  40. package/src/build-dev/vite/evidence-config.d.ts +6 -0
  41. package/src/build-dev/vite/index.js +214 -0
  42. package/src/build-dev/vite/layout-plugins/copy-to-layout.js +71 -0
  43. package/src/build-dev/vite/plugins.js +4 -0
  44. package/src/build-dev/vite/query-directory-hmr.js +32 -0
  45. package/src/build-dev/vite/sidecar-app.js +79 -0
  46. package/src/build-dev/vite/virtuals/QuerySSR.svelte +13 -0
  47. package/src/build-dev/vite/virtuals/__tests/proxyStack.spec.js +52 -0
  48. package/src/build-dev/vite/virtuals/bootstrapContexts.svelte.js +27 -0
  49. package/src/build-dev/vite/virtuals/browser/static-assets.js +3 -0
  50. package/src/build-dev/vite/virtuals/contextKeys.js +3 -0
  51. package/src/build-dev/vite/virtuals/initUsql.js +23 -0
  52. package/src/build-dev/vite/virtuals/node/static-assets.js +56 -0
  53. package/src/build-dev/vite/virtuals/proxyStack.js +37 -0
  54. package/src/build-dev/vite/virtuals/queries.svelte.js +150 -0
  55. package/src/build-dev/vite/virtuals/ssrHook.svelte.js +60 -0
  56. package/src/build-dev/vite/virtuals/updateQueryContext.svelte.js +31 -0
  57. package/src/build-dev/vite/virtuals-dynamic.js +12 -0
  58. package/src/cli.js +5 -0
  59. package/src/cli.root.js +90 -0
  60. package/src/configuration/cli/create.js +20 -0
  61. package/src/configuration/cli/index.js +12 -0
  62. package/src/configuration/createEvidenceConfig.js +51 -0
  63. package/src/configuration/getEvidenceConfig.js +60 -0
  64. package/src/configuration/getEvidenceConfig.legacy.js +27 -0
  65. package/src/configuration/index.js +1 -0
  66. package/src/configuration/schemas/config.schema.js +11 -0
  67. package/src/configuration/schemas/deployment.schema.js +18 -0
  68. package/src/configuration/schemas/deployment.schema.spec.js +34 -0
  69. package/src/configuration/updateEvidenceConfig.js +25 -0
  70. package/src/legacy-compat.js +8 -0
  71. package/src/lib/EvidenceError.js +35 -0
  72. package/src/lib/b64Deep.js +55 -0
  73. package/src/lib/check-import.js +12 -0
  74. package/src/lib/cleanZodErrors.js +22 -0
  75. package/src/lib/cli/cmdFail.js +33 -0
  76. package/src/lib/cli/notices.js +44 -0
  77. package/src/lib/debug.js +40 -0
  78. package/src/lib/flows.d.ts +21 -0
  79. package/src/lib/forceLink.js +28 -0
  80. package/src/lib/history/index.js +70 -0
  81. package/src/lib/history/types.d.ts +8 -0
  82. package/src/lib/pkg-types.helpers.js +16 -0
  83. package/src/lib/processing-queue.js +96 -0
  84. package/src/lib/processing-queue.spec.js +148 -0
  85. package/src/lib/projectPaths.js +29 -0
  86. package/src/lib/projectRoot.js +12 -0
  87. package/src/lib/sharedPromise.js +77 -0
  88. package/src/lib/sharedPromise.spec.js +23 -0
  89. package/src/lib/strict.js +19 -0
  90. package/src/lib/tests/fs.js +21 -0
  91. package/src/lib/tokens.js +7 -0
  92. package/src/lib/unnest-zod-error.js +14 -0
  93. package/src/logger/EvidenceLogger.js +164 -0
  94. package/src/logger/index.js +4 -0
  95. package/src/meta.js +8 -0
  96. package/src/plugins/cli/index.js +13 -0
  97. package/src/plugins/cli/install.js +26 -0
  98. package/src/plugins/cli/install.prompt.js +121 -0
  99. package/src/plugins/components/getAllComponents.js +82 -0
  100. package/src/plugins/components/getComponentsInPlugin.js +33 -0
  101. package/src/plugins/components/loadComponentPlugins.js +48 -0
  102. package/src/plugins/components/loaders/file-loader.js +125 -0
  103. package/src/plugins/components/loaders/manifest-loader.js +53 -0
  104. package/src/plugins/components/schemas/component-manifest.schema.js +5 -0
  105. package/src/plugins/components/types.d.ts +21 -0
  106. package/src/plugins/components/validateOverrides.js +30 -0
  107. package/src/plugins/components/validateOverrides.spec.js +32 -0
  108. package/src/plugins/datasources/Datasources.js +85 -0
  109. package/src/plugins/datasources/Datasources.spec.js +78 -0
  110. package/src/plugins/datasources/SourceResultCache.js +62 -0
  111. package/src/plugins/datasources/buildSourceDirectoryProxy.js +21 -0
  112. package/src/plugins/datasources/cli/edit/Options.fixtures.js +69 -0
  113. package/src/plugins/datasources/cli/edit/Options.js +253 -0
  114. package/src/plugins/datasources/cli/edit/Options.spec.js +496 -0
  115. package/src/plugins/datasources/cli/edit/editSourceConfig.js +201 -0
  116. package/src/plugins/datasources/cli/edit.js +20 -0
  117. package/src/plugins/datasources/cli/edit.prompt.js +100 -0
  118. package/src/plugins/datasources/cli/index.js +18 -0
  119. package/src/plugins/datasources/cli/sources.js +77 -0
  120. package/src/plugins/datasources/evalSources.js +257 -0
  121. package/src/plugins/datasources/getPublishedDatasources.js +62 -0
  122. package/src/plugins/datasources/index.js +6 -0
  123. package/src/plugins/datasources/loadSourceConfig.js +147 -0
  124. package/src/plugins/datasources/loadSourceConfig.spec.js +127 -0
  125. package/src/plugins/datasources/loadSourcePlugins.js +49 -0
  126. package/src/plugins/datasources/loadSources.js +48 -0
  127. package/src/plugins/datasources/schemas/datasource.schema.js +52 -0
  128. package/src/plugins/datasources/schemas/datasourceCache.schema.js +3 -0
  129. package/src/plugins/datasources/schemas/datasourcePlugin.schema.js +37 -0
  130. package/src/plugins/datasources/schemas/datasourcePluginOptions.schema.js +64 -0
  131. package/src/plugins/datasources/schemas/index.js +2 -0
  132. package/src/plugins/datasources/schemas/manifest.schema.js +8 -0
  133. package/src/plugins/datasources/schemas/results.schema.js +126 -0
  134. package/src/plugins/datasources/sub-source-vars.js +55 -0
  135. package/src/plugins/datasources/sub-source-vars.spec.js +46 -0
  136. package/src/plugins/datasources/types.d.ts +53 -0
  137. package/src/plugins/datasources/updateManifest.js +115 -0
  138. package/src/plugins/datasources/updateManifest.spec.js +200 -0
  139. package/src/plugins/datasources/wrapSimpleConnector.js +95 -0
  140. package/src/plugins/datasources/writeSourceConfig.js +60 -0
  141. package/src/plugins/index.js +2 -0
  142. package/src/plugins/layouts/copyMethods/index.js +10 -0
  143. package/src/plugins/layouts/copyMethods/svelte.js +53 -0
  144. package/src/plugins/layouts/copyMethods/sveltekit.js +75 -0
  145. package/src/plugins/layouts/copyToLayout.js +106 -0
  146. package/src/plugins/layouts/loadLayoutPlugin.js +23 -0
  147. package/src/plugins/loadPluginPackage.js +77 -0
  148. package/src/plugins/schemas/index.js +1 -0
  149. package/src/plugins/schemas/plugin-config.schema.js +41 -0
  150. package/src/plugins/schemas/plugin-config.schema.spec.js +21 -0
  151. package/src/plugins/schemas/plugin-package.schema.js +154 -0
  152. package/src/plugins/schemas/plugin-package.schema.spec.js +30 -0
  153. package/src/query-builder.d.ts +1 -0
  154. package/src/query-builder.js +1 -0
  155. package/src/types/duckdb-wellknown.d.ts +35 -0
  156. package/src/types/html-attributes.ts +13 -0
  157. package/src/types/index.d.ts +18 -0
  158. package/src/types/mosaic-sql.d.ts +27 -0
  159. package/src/types/sveltekit-autoimport.d.ts +13 -0
  160. package/src/types/virtuals.d.ts +113 -0
  161. package/src/types.d.ts +1 -0
  162. package/src/usql/index.d.ts +4 -0
  163. package/src/usql/index.js +3 -0
  164. package/src/usql/query/Query.js +1391 -0
  165. package/src/usql/query/Query.spec.js +827 -0
  166. package/src/usql/query/queryScore.js +92 -0
  167. package/src/usql/query/queryScore.spec.js +26 -0
  168. package/src/usql/query/sterilizeQuery.js +89 -0
  169. package/src/usql/query/sterilizeQuery.spec.js +172 -0
  170. package/src/usql/setTrackProxy/setTrackProxy.js +80 -0
  171. package/src/usql/setTrackProxy/setTrackProxy.spec.js +166 -0
  172. package/src/usql/types.ts +86 -0
  173. package/src/usql/utilities/duckdbSerialize.js +29 -0
  174. package/src/usql/utilities/index.js +2 -0
  175. package/src/usql/utilities/resolveMaybePromise.js +30 -0
  176. package/src/utils/batchUp.js +22 -0
  177. package/src/utils/batchUp.spec.js +69 -0
  178. package/src/utils/browserDebounce.js +16 -0
  179. package/src/utils/index.js +7 -0
  180. package/src/utils/svelte/addBasePath.js +74 -0
  181. package/src/utils/svelte/addBasePath.spec.js +108 -0
  182. package/src/utils/svelte/index.js +13 -0
  183. package/src/utils/svelte/inputs.js +138 -0
  184. package/tsconfig.json +22 -0
  185. package/vite.config.js +3 -0
  186. package/vitest.config.js +15 -0
@@ -0,0 +1,115 @@
1
+ import fs from 'fs/promises';
2
+ import path from 'path';
3
+ import { ManifestSchema } from './schemas/manifest.schema.js';
4
+ import merge from 'lodash.merge';
5
+ import chalk from 'chalk';
6
+
7
+ /**
8
+ *
9
+ * @param {string} dataDir
10
+ * @returns
11
+ */
12
+ const safeLoadManifest = async (dataDir) => {
13
+ const manifestPath = path.join(dataDir, 'manifest.json');
14
+ const dirContent = await fs.readdir(dataDir);
15
+ if (!dirContent.includes('manifest.json')) return null;
16
+ if (!(await fs.stat(manifestPath)).isFile()) return null;
17
+ const m = await fs.readFile(manifestPath, 'utf-8');
18
+ try {
19
+ const parsed = JSON.parse(m);
20
+ const valid = ManifestSchema.parse(parsed);
21
+ return valid;
22
+ } catch {
23
+ return null;
24
+ }
25
+ };
26
+
27
+ /**
28
+ * @param {import('./types.js').Manifest} updatedManifest
29
+ * @param {string} dataDir
30
+ * @returns {Promise<import('./types.js').Manifest>}
31
+ */
32
+ export const updateManifest = async (updatedManifest, dataDir) => {
33
+ await fs.mkdir(path.join(dataDir), { recursive: true });
34
+
35
+ const existingManifest = await safeLoadManifest(dataDir);
36
+
37
+ const finalManifest = merge({}, existingManifest); // deep clone
38
+
39
+ if (!updatedManifest.locatedFiles) {
40
+ console.warn(chalk.yellow('[!] No queries were located, operation was a was no-op'));
41
+ return updatedManifest;
42
+ }
43
+ if (!updatedManifest.locatedSchemas) {
44
+ console.warn(chalk.yellow('[!] No sources were located, operation was a was no-op'));
45
+ return updatedManifest;
46
+ }
47
+
48
+ if (existingManifest) {
49
+ for (const schema of updatedManifest.locatedSchemas) {
50
+ console.debug(" Updating schema '" + schema + "'");
51
+ if (!(schema in existingManifest.renderedFiles)) {
52
+ console.debug(' | Schema is new');
53
+ console.debug(
54
+ ` | ${updatedManifest.renderedFiles[schema].length} ${updatedManifest.renderedFiles[schema].length === 1 ? 'query' : 'queries'} will be added`
55
+ );
56
+ finalManifest.renderedFiles[schema] = updatedManifest.renderedFiles[schema];
57
+ continue;
58
+ } else {
59
+ console.debug(' | Schema exists already');
60
+ }
61
+ const locatedQueries = updatedManifest.locatedFiles[schema];
62
+ if (!locatedQueries?.length) {
63
+ console.debug(' | No queries were located, but schema exists already');
64
+ console.debug(' | Previous files will be kept');
65
+ continue;
66
+ }
67
+ console.debug(
68
+ ` | ${locatedQueries.length} ${locatedQueries.length === 1 ? 'query' : 'queries'} found`
69
+ );
70
+ locatedQueries.forEach((q) => console.debug(' | ' + q));
71
+ const newQueries = updatedManifest.renderedFiles[schema].filter((queryPath) => {
72
+ return !existingManifest.renderedFiles[schema].includes(queryPath);
73
+ });
74
+ console.debug(
75
+ ` | ${newQueries.length} ${newQueries.length === 1 ? 'query' : 'queries'} are new`
76
+ );
77
+ newQueries.forEach((q) => console.debug(' | ' + q));
78
+ const existingQueries = existingManifest.renderedFiles[schema]
79
+ .filter((queryPath) => {
80
+ return locatedQueries.includes(path.basename(queryPath, '.parquet'));
81
+ })
82
+ .filter(
83
+ // we want to prefer the new query path
84
+ // file basenames should never conflict, so we can safely use that as a ref
85
+ (queryPath) =>
86
+ !newQueries.some(
87
+ (newQueryPath) =>
88
+ path.basename(newQueryPath, '.parquet') === path.basename(queryPath, '.parquet')
89
+ )
90
+ );
91
+ console.debug(
92
+ ` | ${existingQueries.length} ${existingQueries.length === 1 ? 'query' : 'queries'} already ${existingQueries.length ? 'exists' : 'exist'}`
93
+ );
94
+ existingQueries.forEach((q) => console.debug(' | ' + q));
95
+
96
+ const result = [...existingQueries, ...newQueries].sort((a, b) => a.localeCompare(b));
97
+ console.debug(
98
+ ` | ${result.length} ${result.length === 1 ? 'query' : 'queries'} to be rendered`
99
+ );
100
+ result.forEach((q) => console.debug(' | ' + q));
101
+ finalManifest.renderedFiles[schema] = result;
102
+ }
103
+ for (const schema of Object.keys(existingManifest.renderedFiles)) {
104
+ if (!updatedManifest.locatedSchemas.includes(schema)) {
105
+ console.debug(" Removing schema '" + schema + "'");
106
+ delete finalManifest.renderedFiles[schema];
107
+ }
108
+ }
109
+ } else {
110
+ Object.assign(finalManifest, { renderedFiles: updatedManifest.renderedFiles });
111
+ }
112
+
113
+ await fs.writeFile(path.join(dataDir, 'manifest.json'), JSON.stringify(finalManifest));
114
+ return finalManifest;
115
+ };
@@ -0,0 +1,200 @@
1
+ import { resetFs } from '../../lib/tests/fs.js';
2
+ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
3
+ import fs from 'fs/promises';
4
+ import path from 'path';
5
+
6
+ import { updateManifest } from './updateManifest.js';
7
+
8
+ describe('updateManifest', () => {
9
+ beforeEach(() => {
10
+ resetFs();
11
+ });
12
+ afterEach(() => {
13
+ resetFs();
14
+ });
15
+
16
+ it('should write a manifest to disk when no manifest exists', async () => {
17
+ const manifest = {
18
+ renderedFiles: {
19
+ csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet']
20
+ },
21
+ locatedFiles: {
22
+ csv: ['data', 'nullish']
23
+ },
24
+ locatedSchemas: ['csv']
25
+ };
26
+
27
+ const dataDir = '/_evidence';
28
+ await updateManifest(manifest, dataDir);
29
+
30
+ expect(await fs.stat(path.join(dataDir, 'manifest.json'))).toBeDefined();
31
+ expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
32
+ JSON.stringify({ renderedFiles: manifest.renderedFiles })
33
+ );
34
+ });
35
+
36
+ it('should remove queries that were not updated, but also were not located', async () => {
37
+ const fsManifest = {
38
+ renderedFiles: {
39
+ csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet']
40
+ }
41
+ };
42
+ const updatedManifest = {
43
+ renderedFiles: {
44
+ csv: ['/_evidence/query/csv/data.parquet']
45
+ },
46
+ locatedFiles: {
47
+ csv: ['data']
48
+ },
49
+ locatedSchemas: ['csv']
50
+ };
51
+ const dataDir = '/_evidence';
52
+ await fs.mkdir(path.join(dataDir), { recursive: true });
53
+ await fs.writeFile(
54
+ path.join(dataDir, 'manifest.json'),
55
+ JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
56
+ );
57
+ await updateManifest(updatedManifest, dataDir);
58
+
59
+ expect(await fs.stat(path.join(dataDir, 'manifest.json'))).toBeDefined();
60
+ expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
61
+ JSON.stringify({ renderedFiles: updatedManifest.renderedFiles })
62
+ );
63
+ });
64
+
65
+ it('should not remove queries that were not updated, but were located', async () => {
66
+ const fsManifest = {
67
+ renderedFiles: {
68
+ csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet']
69
+ }
70
+ };
71
+ const updatedManifest = {
72
+ renderedFiles: fsManifest.renderedFiles,
73
+ locatedFiles: {
74
+ csv: ['data', 'nullish']
75
+ },
76
+ locatedSchemas: ['csv']
77
+ };
78
+ const dataDir = '/_evidence';
79
+ await fs.mkdir(path.join(dataDir), { recursive: true });
80
+ await fs.writeFile(
81
+ path.join(dataDir, 'manifest.json'),
82
+ JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
83
+ );
84
+ await updateManifest(updatedManifest, dataDir);
85
+
86
+ expect(await fs.stat(path.join(dataDir, 'manifest.json'))).toBeDefined();
87
+ expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
88
+ JSON.stringify(fsManifest)
89
+ );
90
+ });
91
+
92
+ it('should not touch schemas that were not updated', async () => {
93
+ const fsManifest = {
94
+ renderedFiles: {
95
+ csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet'],
96
+ json: ['/_evidence/query/json/data.parquet']
97
+ }
98
+ };
99
+ const updatedManifest = {
100
+ renderedFiles: {
101
+ csv: ['/_evidence/query/csv/data.parquet']
102
+ },
103
+ locatedFiles: {
104
+ csv: ['data', 'nullish']
105
+ },
106
+ locatedSchemas: ['csv', 'json']
107
+ };
108
+ const dataDir = '/_evidence';
109
+ await fs.mkdir(path.join(dataDir), { recursive: true });
110
+ await fs.writeFile(
111
+ path.join(dataDir, 'manifest.json'),
112
+ JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
113
+ );
114
+ await updateManifest(updatedManifest, dataDir);
115
+
116
+ expect(await fs.stat(path.join(dataDir, 'manifest.json'))).toBeDefined();
117
+ expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
118
+ JSON.stringify(fsManifest)
119
+ );
120
+ });
121
+
122
+ it('should prefer the updated path for a query', async () => {
123
+ const fsManifest = {
124
+ renderedFiles: {
125
+ csv: ['/_evidence/query/csv/data.parquet', '/_evidence/query/csv/nullish.parquet']
126
+ }
127
+ };
128
+ const updatedManifest = {
129
+ renderedFiles: {
130
+ csv: ['/__evidence/query/csv/data.parquet', '/__evidence/query/csv/nullish.parquet']
131
+ },
132
+ locatedFiles: {
133
+ csv: ['data', 'nullish']
134
+ },
135
+ locatedSchemas: ['csv']
136
+ };
137
+ const dataDir = '/_evidence';
138
+ await fs.mkdir(path.join(dataDir), { recursive: true });
139
+ await fs.writeFile(
140
+ path.join(dataDir, 'manifest.json'),
141
+ JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
142
+ );
143
+ await updateManifest(updatedManifest, dataDir);
144
+ expect(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8')).toEqual(
145
+ JSON.stringify({
146
+ renderedFiles: updatedManifest.renderedFiles
147
+ })
148
+ );
149
+ });
150
+ it('should merge a new source correctly', async () => {
151
+ const fsManifest = {
152
+ renderedFiles: {
153
+ needful_things: ['static/data/needful_things/orders/orders.parquet']
154
+ }
155
+ };
156
+ const updatedManifest = {
157
+ renderedFiles: {
158
+ my_csv: ['static/data/my_csv/x/x.parquet'],
159
+ needful_things: ['static/data/needful_things/orders/orders.parquet']
160
+ },
161
+ locatedFiles: {
162
+ my_csv: ['x'],
163
+ needful_things: ['needful_things', 'orders']
164
+ },
165
+ locatedSchemas: ['my_csv', 'needful_things']
166
+ };
167
+ const dataDir = '/_evidence';
168
+ await fs.mkdir(path.join(dataDir), { recursive: true });
169
+ await fs.writeFile(
170
+ path.join(dataDir, 'manifest.json'),
171
+ JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
172
+ );
173
+ await updateManifest(updatedManifest, dataDir);
174
+ expect(JSON.parse(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8'))).toEqual({
175
+ renderedFiles: updatedManifest.renderedFiles
176
+ });
177
+ });
178
+ it('should remove an old source correctly', async () => {
179
+ const fsManifest = {
180
+ renderedFiles: {
181
+ needful_things: ['static/data/needful_things/orders/orders.parquet']
182
+ }
183
+ };
184
+ const updatedManifest = {
185
+ renderedFiles: {},
186
+ locatedFiles: {},
187
+ locatedSchemas: []
188
+ };
189
+ const dataDir = '/_evidence';
190
+ await fs.mkdir(path.join(dataDir), { recursive: true });
191
+ await fs.writeFile(
192
+ path.join(dataDir, 'manifest.json'),
193
+ JSON.stringify({ renderedFiles: fsManifest.renderedFiles })
194
+ );
195
+ await updateManifest(updatedManifest, dataDir);
196
+ expect(JSON.parse(await fs.readFile(path.join(dataDir, 'manifest.json'), 'utf8'))).toEqual({
197
+ renderedFiles: updatedManifest.renderedFiles
198
+ });
199
+ });
200
+ });
@@ -0,0 +1,95 @@
1
+ import fs from 'fs/promises';
2
+ import { EvidenceError } from '../../lib/EvidenceError.js';
3
+ import path from 'path';
4
+ import { readFileSync, statSync } from 'fs';
5
+ import chalk from 'chalk';
6
+
7
+ /**
8
+ * @param {import("./Datasources.js").Datasource} mod
9
+ * @param {import('./schemas/datasource.schema.js').DatasourceSpec & {dir: string}} source
10
+ * @returns {import('./types.js').ProcessSourceFn}
11
+ */
12
+ export const wrapSimpleConnector = (mod, source) => {
13
+ if (!('getRunner' in mod))
14
+ throw new EvidenceError(
15
+ 'Internal Error',
16
+ 'Advanced connector was passed to wrapSimpleConnector. This should not occur'
17
+ );
18
+ /**
19
+ * @type {import('./types.js').ProcessSourceFn}
20
+ */
21
+ return async function* (...args) {
22
+ const [, , utils] = args;
23
+ const runner = await mod.getRunner(source.options, source.dir);
24
+ // TODO: Use fsproxy instead of fs
25
+ /**
26
+ *
27
+ * @param {string} dir
28
+ * @returns {AsyncGenerator<import('./types.js').QueryResultTable | EvidenceError>}
29
+ */
30
+ const processDir = async function* (dir) {
31
+ const sourceFiles = await fs.readdir(dir, { withFileTypes: true });
32
+ for (const sourceFile of sourceFiles) {
33
+ // Why is the dirent interface so unstable?
34
+ // This behaves differently depending on which minor version of 18 / 20 you are using
35
+ const dirPath =
36
+ // @ts-expect-error
37
+ 'parentPath' in sourceFile ? sourceFile.parentPath + '' : (sourceFile.path ?? dir);
38
+ if (sourceFile.name === 'connection.yaml' || sourceFile.name === 'connection.options.yaml')
39
+ continue;
40
+ if (sourceFile.isDirectory()) {
41
+ yield* processDir(path.join(dirPath, sourceFile.name));
42
+ continue;
43
+ }
44
+
45
+ if (!sourceFile.isFile()) continue;
46
+
47
+ const sourceFilePath = path.join(dirPath, sourceFile.name);
48
+ const sourceFileName = sourceFile.name.split('.').at(0) ?? '';
49
+
50
+ const stat = statSync(sourceFilePath);
51
+ let sourceFileContent;
52
+ if (stat.size > 1024 * 1024 * 32 /* 32 Megabytes */) {
53
+ console.debug(chalk.dim('Will not eagerly load files larger than 32 Megabytes.'));
54
+ sourceFileContent = '';
55
+ } else {
56
+ sourceFileContent = readFileSync(sourceFilePath, 'utf-8');
57
+ }
58
+
59
+ // Check if it should be skipped, and return an empty result (this is handled above)
60
+ if (!utils.shouldRun(sourceFileName, sourceFileContent)) {
61
+ yield utils.escape(sourceFileName, sourceFileContent);
62
+ continue;
63
+ }
64
+
65
+ try {
66
+ yield {
67
+ name: /** @type {string} */ (sourceFileName),
68
+ content: sourceFileContent,
69
+ columnTypes: [],
70
+ ...(await runner(
71
+ utils.subSourceVariables(sourceFileContent),
72
+ sourceFilePath,
73
+ source.buildOptions.batchSize ?? 1000 * 1000 // TODO: BatchSize configurable? Perhaps per-source plugin or per connection
74
+ ))
75
+ };
76
+ } catch (e) {
77
+ let message = JSON.stringify(e);
78
+ if (e instanceof Error) message = e.message;
79
+ if (e instanceof EvidenceError) {
80
+ e.metadata.tableName = sourceFileName;
81
+ yield e;
82
+ }
83
+ yield new EvidenceError(
84
+ message,
85
+ [`${source.name}/${sourceFileName ?? ''} failed to execute`],
86
+ { cause: e },
87
+ { tableName: sourceFileName }
88
+ );
89
+ }
90
+ }
91
+ };
92
+
93
+ yield* processDir(source.dir);
94
+ };
95
+ };
@@ -0,0 +1,60 @@
1
+ import yaml from 'yaml';
2
+ import path from 'path';
3
+ import fs from 'fs/promises';
4
+ import { getSafeOptions, getSecretOptions } from './cli/edit/Options.js';
5
+ import { encodeBase64Deep } from '../../lib/b64Deep.js';
6
+ import { sourcesDirectory } from '../../lib/projectPaths.js';
7
+
8
+ /**
9
+ *
10
+ * @param {import("./cli/edit/Options.js").OptionsObj} options
11
+ * @param {import("./schemas/datasource.schema.js").DatasourceSpec & { dir?: string }} source
12
+ * @param {import('../../lib/flows.js').EvidenceLoadIndicator} [spinner]
13
+ * @returns {Promise<import("./schemas/datasource.schema.js").DatasourceSpec & { dir: string }>} Source, with the directory attached if it was not previously
14
+ */
15
+ export const writeSourceConfig = async (options, source, spinner) => {
16
+ const secrets = getSecretOptions(options);
17
+ const safes = getSafeOptions(options);
18
+
19
+ const safesFileString = [
20
+ `# This file was automatically generated`,
21
+ yaml.stringify({ ...source, options: safes, dir: undefined })
22
+ ]
23
+ .join('\n')
24
+ .replace(
25
+ 'buildOptions:',
26
+ [`# Advanced Options, you probably don't want to change this`, `buildOptions:`].join('\n')
27
+ );
28
+
29
+ const secretsFileString = [
30
+ `# This file was automatically generated`,
31
+ `# Do not edit this file manually; instead use npx evidence-sdk sources edit`,
32
+ ``,
33
+ `# It should *not* be source controlled, as it likely contain credentials or other sensitive configuration values.`,
34
+ `# Values in this file are base64 encoded; https://it-tools.tech/base64-string-converter has an excellent encoder / decoder tool.`,
35
+ `# Base64 is NOT encryption, and should not be treated as secure`,
36
+ yaml.stringify(encodeBase64Deep(secrets))
37
+ ].join('\n');
38
+
39
+ if (
40
+ !source.dir ||
41
+ !path.dirname(source.dir).startsWith(sourcesDirectory) ||
42
+ !path.isAbsolute(source.dir)
43
+ ) {
44
+ source.dir = path.join(sourcesDirectory, source.name);
45
+ }
46
+
47
+ const safesPath = path.join(source.dir, 'connection.yaml');
48
+ const secretsPath = path.join(source.dir, 'connection.options.yaml');
49
+
50
+ await fs.mkdir(source.dir, { recursive: true });
51
+
52
+ spinner?.start(`Saving ${source.name}`);
53
+ await fs.writeFile(safesPath, safesFileString);
54
+ await fs.writeFile(secretsPath, secretsFileString);
55
+ spinner?.stop(`Finished saving ${source.name}`);
56
+
57
+ return /** @type {import("./schemas/datasource.schema.js").DatasourceSpec & { dir: string }} */ (
58
+ source
59
+ );
60
+ };
@@ -0,0 +1,2 @@
1
+ export * from './datasources/index.js';
2
+ export * from './schemas/index.js';
@@ -0,0 +1,10 @@
1
+ import * as sveltekit from './sveltekit.js';
2
+ import * as svelte from './svelte.js';
3
+
4
+ /**
5
+ * @type {Record<string, { copyAll: (rootDir: string, rootTargetDir: string) => Promise<void>, copyFile: (rootDir: string, rootTargetDir: string, file: import("fs").Dirent) => void }>}
6
+ */
7
+ export const copyMethods = {
8
+ sveltekit: sveltekit,
9
+ svelte: svelte
10
+ };
@@ -0,0 +1,53 @@
1
+ import path from 'path';
2
+ import fs from 'fs/promises';
3
+ import chalk from 'chalk';
4
+
5
+ /**
6
+ *
7
+ * @param {string} rootDir
8
+ * @param {string} rootTargetDir
9
+ * @param {import("fs").Dirent} file
10
+ */
11
+ const getTargets = async (rootDir, rootTargetDir, file) => {
12
+ const relativePath = path.relative(path.resolve(rootDir), path.resolve(file.path, file.name));
13
+ const absolutePath = path.join(rootTargetDir, relativePath);
14
+
15
+ return {
16
+ relativePath,
17
+ absolutePath
18
+ };
19
+ };
20
+
21
+ /**
22
+ * @param {string} rootDir
23
+ * @param {string} rootTargetDir
24
+ * @param {import("fs").Dirent} page
25
+ */
26
+ export const copyFile = async (rootDir, rootTargetDir, page) => {
27
+ await fs.mkdir(rootTargetDir, { recursive: true });
28
+ const { relativePath } = await getTargets(rootDir, rootTargetDir, page);
29
+ const p = path.parse(path.join(rootTargetDir, relativePath));
30
+ const from = path.join(page.path, page.name);
31
+ let to = path.join(p.dir, page.name);
32
+ await fs.cp(from, to);
33
+ console.debug(chalk.dim(`Copied ${from} -> ${to}`));
34
+ };
35
+
36
+ /**
37
+ * @param {string} rootDir
38
+ * @param {string} rootTargetDir
39
+ */
40
+ export const copyAll = async (rootDir, rootTargetDir) => {
41
+ await fs.mkdir(rootTargetDir, { recursive: true });
42
+
43
+ const pages = await fs.readdir(path.resolve(rootDir), { recursive: true, withFileTypes: true });
44
+
45
+ for (const page of pages) {
46
+ const { absolutePath } = await getTargets(rootDir, rootTargetDir, page);
47
+ if (page.isDirectory()) {
48
+ await fs.mkdir(absolutePath, { recursive: true });
49
+ } else {
50
+ await copyFile(rootDir, rootTargetDir, page);
51
+ }
52
+ }
53
+ };
@@ -0,0 +1,75 @@
1
+ import path from 'path';
2
+ import fs from 'fs/promises';
3
+ import chalk from 'chalk';
4
+ import { projectRoot } from '../../../lib/projectRoot.js';
5
+
6
+ /**
7
+ *
8
+ * @param {string} rootDir
9
+ * @param {string} rootTargetDir
10
+ * @param {import("fs").Dirent} file
11
+ */
12
+ const getTargets = async (rootDir, rootTargetDir, file) => {
13
+ const relativePath = path.relative(path.resolve(rootDir), path.resolve(file.path, file.name));
14
+ const absolutePath = path.join(rootTargetDir, relativePath);
15
+
16
+ return {
17
+ relativePath,
18
+ absolutePath
19
+ };
20
+ };
21
+
22
+ /**
23
+ * @param {string} rootDir
24
+ * @param {string} rootTargetDir
25
+ * @param {import("fs").Dirent} page
26
+ */
27
+ export const copyFile = async (rootDir, rootTargetDir, page) => {
28
+ // TODO: Error handling
29
+ if (!rootTargetDir.includes('(evidence-pages)'))
30
+ rootTargetDir = path.join(rootTargetDir, '(evidence-pages)');
31
+ await fs.mkdir(rootTargetDir, { recursive: true });
32
+
33
+ const { relativePath } = await getTargets(rootDir, rootTargetDir, page);
34
+ const p = path.parse(path.join(rootTargetDir, relativePath));
35
+ const filename = page.name.split('.')[0]; // rip off the filename
36
+ const extention = page.name.split('.').slice(1).join('.'); // rip off the filename
37
+ const from = path.join(page.path, page.name);
38
+ let to;
39
+ if (page.name === '+layout.svelte') {
40
+ to = path.join(p.dir, page.name);
41
+ } else if (page.name.split('.')[0] === 'index') {
42
+ to = path.join(p.dir, `+page.${extention}`);
43
+ } else {
44
+ // needs a dir
45
+ // same as relative, excludes file extension
46
+ const dirname = path.join(
47
+ rootTargetDir,
48
+ path.relative(path.resolve(projectRoot, 'pages'), path.resolve(page.path, filename))
49
+ );
50
+ await fs.mkdir(dirname, { recursive: true });
51
+ to = path.join(dirname, `+page.${extention}`);
52
+ }
53
+ await fs.cp(from, to);
54
+ console.debug(chalk.dim(`Copied ${from} -> ${to}`));
55
+ };
56
+
57
+ /**
58
+ * @param {string} rootDir
59
+ * @param {string} rootTargetDir
60
+ */
61
+ export const copyAll = async (rootDir, rootTargetDir) => {
62
+ rootTargetDir = path.join(rootTargetDir, '(evidence-pages)');
63
+ await fs.mkdir(rootTargetDir, { recursive: true });
64
+
65
+ const pages = await fs.readdir(path.resolve(rootDir), { recursive: true, withFileTypes: true });
66
+
67
+ for (const page of pages) {
68
+ const { absolutePath } = await getTargets(rootDir, rootTargetDir, page);
69
+ if (page.isDirectory()) {
70
+ await fs.mkdir(absolutePath, { recursive: true });
71
+ } else {
72
+ await copyFile(rootDir, rootTargetDir, page);
73
+ }
74
+ }
75
+ };