duckfn-docs-kit 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +689 -0
- package/LICENSE +21 -0
- package/README.md +107 -0
- package/dist/dom.d.ts +69 -0
- package/dist/home/DfkFeatures.d.ts +20 -0
- package/dist/home/DfkHero.d.ts +25 -0
- package/dist/home/DfkNextSteps.d.ts +16 -0
- package/dist/home/styles.d.ts +8 -0
- package/dist/index.d.ts +50 -0
- package/dist/index.js +2 -0
- package/dist/register-DKLiYs-F.js +2324 -0
- package/dist/register.d.ts +10 -0
- package/dist/remark.d.ts +21 -0
- package/dist/remark.js +15 -0
- package/dist/runtimeConfig-Bokbb8VH.js +106 -0
- package/dist/sql/DfkSql.d.ts +7 -0
- package/dist/sql/PreviewTabs.d.ts +37 -0
- package/dist/sql/client.d.ts +1 -0
- package/dist/sql/client.js +4 -0
- package/dist/sql/editor.d.ts +16 -0
- package/dist/sql/extensions.d.ts +108 -0
- package/dist/sql/extensions.js +198 -0
- package/dist/sql/remark.d.ts +88 -0
- package/dist/sql/remark.js +69 -0
- package/dist/sql/renderers.d.ts +44 -0
- package/dist/sql/runtime.d.ts +105 -0
- package/dist/sql/runtimeConfig.d.ts +80 -0
- package/dist/sql/styles.d.ts +6 -0
- package/dist/toc-toggle/TocToggle.d.ts +46 -0
- package/dist/toc-toggle/TocToggle.js +69 -0
- package/dist/toc-toggle/client.d.ts +1 -0
- package/dist/toc-toggle/client.js +9 -0
- package/dist/toc-toggle/plugin.d.ts +36 -0
- package/dist/toc-toggle/plugin.js +13 -0
- package/dist/types.d.ts +42 -0
- package/package.json +73 -0
- package/src/dom.ts +109 -0
- package/src/home/DfkFeatures.ts +78 -0
- package/src/home/DfkHero.ts +128 -0
- package/src/home/DfkNextSteps.ts +73 -0
- package/src/home/home.css +520 -0
- package/src/home/styles.ts +28 -0
- package/src/index.ts +59 -0
- package/src/kit.css +19 -0
- package/src/register.ts +39 -0
- package/src/remark.ts +60 -0
- package/src/sql/DfkSql.css +226 -0
- package/src/sql/DfkSql.ts +620 -0
- package/src/sql/PreviewTabs.ts +169 -0
- package/src/sql/client.ts +16 -0
- package/src/sql/editor.ts +75 -0
- package/src/sql/extensions.ts +470 -0
- package/src/sql/remark.ts +213 -0
- package/src/sql/renderers.ts +916 -0
- package/src/sql/runtime.ts +348 -0
- package/src/sql/runtimeConfig.ts +249 -0
- package/src/sql/sql.css +397 -0
- package/src/sql/styles.ts +24 -0
- package/src/theme/tokens.css +75 -0
- package/src/toc-toggle/TocToggle.css +69 -0
- package/src/toc-toggle/TocToggle.ts +172 -0
- package/src/toc-toggle/client.ts +20 -0
- package/src/toc-toggle/plugin.ts +54 -0
- package/src/types.ts +47 -0
- package/src/vite-env.d.ts +8 -0
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
import {createHash} from 'node:crypto';
|
|
2
|
+
import {mkdir, readFile, rename, stat, unlink, writeFile} from 'node:fs/promises';
|
|
3
|
+
import {createRequire} from 'node:module';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import {
|
|
6
|
+
DFK_SQL_RUNTIME_TAG_ID,
|
|
7
|
+
isAbsoluteHttpUrl,
|
|
8
|
+
normalizePreloadEntry,
|
|
9
|
+
type PreloadEntry,
|
|
10
|
+
type PreloadReleaseSource,
|
|
11
|
+
type SiteRuntimeConfig,
|
|
12
|
+
} from './runtimeConfig';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* `duckfn-docs-kit/sql/extensions` — the Docusaurus plugin behind the
|
|
16
|
+
* runnable-SQL examples' extensions.
|
|
17
|
+
*
|
|
18
|
+
* It does two things, both driven by one ordered `preload` list configured in
|
|
19
|
+
* `docusaurus.config.ts`:
|
|
20
|
+
*
|
|
21
|
+
* 1. At startup (dev server and build alike) it fetches every `release`
|
|
22
|
+
* source from that GitHub repository's **latest** release into the site's
|
|
23
|
+
* `static/` directory, where the file is then served same-origin.
|
|
24
|
+
* Downloads are cached locally and only re-fetched when the release
|
|
25
|
+
* asset's sha256 differs (GitHub's own `digest` field is the comparison).
|
|
26
|
+
* 2. It injects the resolved preload list as one JSON `<script>` tag into
|
|
27
|
+
* every page; the browser runtime (`sql/runtime`) reads it once and loads
|
|
28
|
+
* the extensions in order while the DuckDB instance initialises.
|
|
29
|
+
*
|
|
30
|
+
* It also injects the kit's client bootstrap (`sql/client.ts`) on every page
|
|
31
|
+
* through `getClientModules()`: docs pages never import the kit's React tree,
|
|
32
|
+
* so the `dfk-*` element registration has to come from a client module — and
|
|
33
|
+
* it comes from here, not from a file the site keeps by hand.
|
|
34
|
+
*
|
|
35
|
+
* The site-relative `url` of an entry doubles as the fetch destination *and*
|
|
36
|
+
* the runtime path, so the two can never drift: with baseUrl `/duckfn/`,
|
|
37
|
+
* `duckdb-extensions/duckfn.duckdb_extension.wasm` lands in
|
|
38
|
+
* `<siteDir>/static/duckdb-extensions/duckfn.duckdb_extension.wasm` and is
|
|
39
|
+
* preloaded from `/duckfn/duckdb-extensions/duckfn.duckdb_extension.wasm`.
|
|
40
|
+
* (Docusaurus serves `static/` under each locale's baseUrl, so the localized
|
|
41
|
+
* value from the plugin context is the right prefix for every build.)
|
|
42
|
+
*
|
|
43
|
+
* This is Node-side build code: it must not import any browser module, and
|
|
44
|
+
* the browser side must not import this file (the shared contract lives in
|
|
45
|
+
* `./runtimeConfig`).
|
|
46
|
+
*/
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* The slice of Docusaurus' plugin API this module touches, typed structurally
|
|
50
|
+
* instead of importing `@docusaurus/types`: the kit stays free of Docusaurus
|
|
51
|
+
* dependencies, and the consuming site's own typecheck proves compatibility
|
|
52
|
+
* when the returned module lands in its `plugins` list.
|
|
53
|
+
*/
|
|
54
|
+
export interface DfkExtensionsContext {
|
|
55
|
+
siteDir: string;
|
|
56
|
+
siteConfig: {baseUrl: string};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
interface DfkHtmlTag {
|
|
60
|
+
tagName: string;
|
|
61
|
+
attributes: Record<string, string>;
|
|
62
|
+
innerHTML: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface DfkExtensionsPlugin {
|
|
66
|
+
name: string;
|
|
67
|
+
getClientModules(): string[];
|
|
68
|
+
loadContent(): Promise<void>;
|
|
69
|
+
injectHtmlTags(): {headTags: DfkHtmlTag[]};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The plugin module Docusaurus calls with its `LoadContext` and options. */
|
|
73
|
+
export type DfkExtensionsPluginModule = (context: DfkExtensionsContext) => DfkExtensionsPlugin;
|
|
74
|
+
|
|
75
|
+
/** Options for {@link dfkExtensions}. */
|
|
76
|
+
export interface DfkExtensionsOptions {
|
|
77
|
+
/**
|
|
78
|
+
* The ordered list of extensions every page preloads while the shared
|
|
79
|
+
* DuckDB instance initialises — official names, `{name, repository}`
|
|
80
|
+
* entries, or `{url}` files (optionally fetched from a GitHub release).
|
|
81
|
+
* See `PreloadEntry` in `./runtimeConfig` for the exact shapes.
|
|
82
|
+
*/
|
|
83
|
+
preload?: PreloadEntry[];
|
|
84
|
+
/**
|
|
85
|
+
* Let `LOAD` accept extensions whose signature does not verify. Needed
|
|
86
|
+
* whenever a preloaded file is not signed with DuckDB's keys — the GitHub
|
|
87
|
+
* release assets of a third-party extension are not — and merged with the
|
|
88
|
+
* per-block setting of whichever block initialises the runtime first.
|
|
89
|
+
*/
|
|
90
|
+
allowUnsignedExtensions?: boolean;
|
|
91
|
+
/**
|
|
92
|
+
* Where release assets are cached between builds. Relative paths resolve
|
|
93
|
+
* against the Docusaurus site directory; defaults to
|
|
94
|
+
* `<siteDir>/.cache/duckfn-docs-kit`.
|
|
95
|
+
*/
|
|
96
|
+
cacheDir?: string;
|
|
97
|
+
/** A GitHub token for the release API (rate limits, private repositories). Defaults to `process.env.GITHUB_TOKEN`. */
|
|
98
|
+
token?: string;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const LOG_PREFIX = '[dfk-extensions] ';
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Builds the Docusaurus plugin. Usage in `docusaurus.config.ts`:
|
|
105
|
+
*
|
|
106
|
+
* ```ts
|
|
107
|
+
* import {dfkExtensions} from 'duckfn-docs-kit/sql/extensions';
|
|
108
|
+
*
|
|
109
|
+
* plugins: [
|
|
110
|
+
* dfkExtensions({
|
|
111
|
+
* allowUnsignedExtensions: true,
|
|
112
|
+
* preload: [
|
|
113
|
+
* {name: 'inet'},
|
|
114
|
+
* {
|
|
115
|
+
* url: 'duckdb-extensions/duckfn.duckdb_extension.wasm',
|
|
116
|
+
* release: {repository: 'shijianjs/duckfn', asset: 'duckfn-wasm_eh.duckdb_extension.wasm'},
|
|
117
|
+
* },
|
|
118
|
+
* ],
|
|
119
|
+
* }),
|
|
120
|
+
* ],
|
|
121
|
+
* ```
|
|
122
|
+
*/
|
|
123
|
+
export function dfkExtensions(options: DfkExtensionsOptions = {}): DfkExtensionsPluginModule {
|
|
124
|
+
const {
|
|
125
|
+
allowUnsignedExtensions = false,
|
|
126
|
+
preload = [],
|
|
127
|
+
cacheDir,
|
|
128
|
+
token = process.env.GITHUB_TOKEN,
|
|
129
|
+
} = options;
|
|
130
|
+
|
|
131
|
+
// Validated once, however the lifecycles are entered, so a malformed entry
|
|
132
|
+
// fails the build before any download is attempted.
|
|
133
|
+
let prepared: SiteRuntimeConfig | null = null;
|
|
134
|
+
const prepare = (): SiteRuntimeConfig => {
|
|
135
|
+
prepared ??= {
|
|
136
|
+
...(allowUnsignedExtensions ? {allowUnsignedExtensions: true} : {}),
|
|
137
|
+
preload: preload.map((entry, index) => {
|
|
138
|
+
try {
|
|
139
|
+
return normalizePreloadEntry(entry);
|
|
140
|
+
} catch (error) {
|
|
141
|
+
throw new Error(`duckfn-docs-kit preload[${index}]: ${messageOf(error)}`);
|
|
142
|
+
}
|
|
143
|
+
}),
|
|
144
|
+
};
|
|
145
|
+
return prepared;
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
return (context) => ({
|
|
149
|
+
name: 'dfk-extensions',
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* The `dfk-*` element registration (see `sql/client.ts`), injected on
|
|
153
|
+
* every page so docs pages — whose React tree never imports the kit —
|
|
154
|
+
* still upgrade the runnable-SQL elements.
|
|
155
|
+
*/
|
|
156
|
+
getClientModules() {
|
|
157
|
+
// Resolved from the consuming site, so the config bundler cannot break
|
|
158
|
+
// the lookup; the exports map (`./*` -> dist) keeps the subpath valid
|
|
159
|
+
// even if the entry moves.
|
|
160
|
+
const requireFromSite = createRequire(path.join(context.siteDir, 'package.json'));
|
|
161
|
+
return [requireFromSite.resolve('duckfn-docs-kit/sql/client')];
|
|
162
|
+
},
|
|
163
|
+
|
|
164
|
+
async loadContent() {
|
|
165
|
+
const config = prepare();
|
|
166
|
+
const {siteDir} = context;
|
|
167
|
+
const cacheRoot = path.resolve(siteDir, cacheDir ?? '.cache/duckfn-docs-kit');
|
|
168
|
+
for (const entry of config.preload) {
|
|
169
|
+
if (typeof entry === 'string' || !('url' in entry)) {
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
const dest = path.join(siteDir, 'static', entry.url);
|
|
173
|
+
const label = `static/${entry.url}`;
|
|
174
|
+
if (entry.release) {
|
|
175
|
+
await fetchReleaseAsset(entry.release, {dest, label, cacheRoot, token});
|
|
176
|
+
} else if (!isAbsoluteHttpUrl(entry.url) && !(await fileExists(dest))) {
|
|
177
|
+
throw new Error(
|
|
178
|
+
`${LOG_PREFIX}preload url "${entry.url}" has no file: place the file under ` +
|
|
179
|
+
`static/${entry.url}, or point the entry at a GitHub release source`,
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
|
|
185
|
+
injectHtmlTags() {
|
|
186
|
+
const config = prepare();
|
|
187
|
+
if (!config.allowUnsignedExtensions && config.preload.length === 0) {
|
|
188
|
+
return {headTags: []};
|
|
189
|
+
}
|
|
190
|
+
const resolved: SiteRuntimeConfig = {
|
|
191
|
+
...config,
|
|
192
|
+
preload: config.preload.map((entry) => resolveForSite(entry, context.siteConfig.baseUrl)),
|
|
193
|
+
};
|
|
194
|
+
return {
|
|
195
|
+
headTags: [
|
|
196
|
+
{
|
|
197
|
+
tagName: 'script',
|
|
198
|
+
attributes: {id: DFK_SQL_RUNTIME_TAG_ID, type: 'application/json'},
|
|
199
|
+
// `<` is escaped so no config string can ever close the script tag.
|
|
200
|
+
innerHTML: JSON.stringify(resolved).replaceAll('<', '\\u003c'),
|
|
201
|
+
},
|
|
202
|
+
],
|
|
203
|
+
};
|
|
204
|
+
},
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Fetches one release asset into `dest`, going through the local cache. */
|
|
209
|
+
async function fetchReleaseAsset(
|
|
210
|
+
source: PreloadReleaseSource,
|
|
211
|
+
options: {dest: string; label: string; cacheRoot: string; token?: string},
|
|
212
|
+
): Promise<void> {
|
|
213
|
+
const {dest, label, cacheRoot, token} = options;
|
|
214
|
+
const log = (text: string): void => console.log(`${LOG_PREFIX}${source.repository}#${source.asset}: ${text}`);
|
|
215
|
+
const resolved = await resolveReleaseAsset(source, {cacheRoot, token, log});
|
|
216
|
+
await writeIfChanged(dest, resolved.bytes, resolved.sha256, log, label);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Resolves the asset's bytes: cache hit when the release digest matches,
|
|
221
|
+
* otherwise download + verify + re-cache. A network failure falls back to a
|
|
222
|
+
* cached copy with a warning — offline development keeps working — but only
|
|
223
|
+
* when a cache exists, so CI (which starts empty) fails loudly.
|
|
224
|
+
*/
|
|
225
|
+
async function resolveReleaseAsset(
|
|
226
|
+
source: PreloadReleaseSource,
|
|
227
|
+
options: {cacheRoot: string; token?: string; log: (text: string) => void},
|
|
228
|
+
): Promise<CachedAsset> {
|
|
229
|
+
const {cacheRoot, token, log} = options;
|
|
230
|
+
const cachePaths = cacheFilePaths(cacheRoot, source);
|
|
231
|
+
const cached = await readCachedAsset(cachePaths);
|
|
232
|
+
const useCached = (error: unknown): CachedAsset => {
|
|
233
|
+
if (!cached) {
|
|
234
|
+
throw error;
|
|
235
|
+
}
|
|
236
|
+
console.warn(
|
|
237
|
+
`${LOG_PREFIX}${source.repository}#${source.asset}: ${messageOf(error)} — ` +
|
|
238
|
+
`using the cached copy (${shortHash(cached.sha256)})`,
|
|
239
|
+
);
|
|
240
|
+
return cached;
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
let release: GithubRelease;
|
|
244
|
+
try {
|
|
245
|
+
release = await fetchLatestRelease(source.repository, token);
|
|
246
|
+
} catch (error) {
|
|
247
|
+
return useCached(error);
|
|
248
|
+
}
|
|
249
|
+
const asset = pickAsset(release, source);
|
|
250
|
+
const expected = digestSha256(asset.digest);
|
|
251
|
+
if (cached && expected !== null && cached.sha256 === expected) {
|
|
252
|
+
log(`up to date (${shortHash(expected)})`);
|
|
253
|
+
return cached;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
log(`downloading${release.tag_name ? ` ${release.tag_name}` : ''}…`);
|
|
257
|
+
let bytes: Buffer;
|
|
258
|
+
try {
|
|
259
|
+
bytes = await downloadAsset(asset, token);
|
|
260
|
+
} catch (error) {
|
|
261
|
+
return useCached(error);
|
|
262
|
+
}
|
|
263
|
+
const sha256 = sha256Hex(bytes);
|
|
264
|
+
if (expected !== null && sha256 !== expected) {
|
|
265
|
+
// Louder than the cache fallback on purpose: the bytes on offer do not
|
|
266
|
+
// match what GitHub says the release asset is.
|
|
267
|
+
throw new Error(
|
|
268
|
+
`${LOG_PREFIX}${source.repository}#${source.asset}: sha256 mismatch, ` +
|
|
269
|
+
`expected ${expected}, downloaded ${sha256}`,
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
const tag = release.tag_name ?? '';
|
|
273
|
+
await writeCachedAsset(cachePaths, bytes, sha256, tag);
|
|
274
|
+
log(`cached ${shortHash(sha256)}`);
|
|
275
|
+
return {bytes, sha256, tag};
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/** One GitHub release asset, reduced to the fields this plugin reads. */
|
|
279
|
+
interface GithubAsset {
|
|
280
|
+
name?: string;
|
|
281
|
+
browser_download_url?: string;
|
|
282
|
+
digest?: string | null;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
interface GithubRelease {
|
|
286
|
+
tag_name?: string;
|
|
287
|
+
assets?: GithubAsset[];
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
const GITHUB_API = 'https://api.github.com';
|
|
291
|
+
|
|
292
|
+
async function fetchLatestRelease(repository: string, token: string | undefined): Promise<GithubRelease> {
|
|
293
|
+
const headers = githubHeaders(token, 'application/vnd.github+json');
|
|
294
|
+
const response = await fetch(`${GITHUB_API}/repos/${repository}/releases/latest`, {headers});
|
|
295
|
+
if (!response.ok) {
|
|
296
|
+
throw new Error(`GitHub API ${response.status} for ${repository}: ${(await response.text()).slice(0, 200)}`);
|
|
297
|
+
}
|
|
298
|
+
return (await response.json()) as GithubRelease;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function pickAsset(release: GithubRelease, source: PreloadReleaseSource): GithubAsset {
|
|
302
|
+
const assets = release.assets ?? [];
|
|
303
|
+
const asset = assets.find((candidate) => candidate.name === source.asset);
|
|
304
|
+
if (!asset || typeof asset.browser_download_url !== 'string') {
|
|
305
|
+
const available = assets.map((candidate) => candidate.name).filter(Boolean).join(', ') || 'none';
|
|
306
|
+
throw new Error(
|
|
307
|
+
`${LOG_PREFIX}release ${release.tag_name ?? '?'} of ${source.repository} has no asset ` +
|
|
308
|
+
`"${source.asset}" (available: ${available})`,
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
return asset;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async function downloadAsset(asset: GithubAsset, token: string | undefined): Promise<Buffer> {
|
|
315
|
+
const response = await fetch(asset.browser_download_url!, {
|
|
316
|
+
headers: githubHeaders(token, 'application/octet-stream'),
|
|
317
|
+
redirect: 'follow',
|
|
318
|
+
});
|
|
319
|
+
if (!response.ok) {
|
|
320
|
+
throw new Error(`asset download failed with ${response.status}: ${asset.browser_download_url}`);
|
|
321
|
+
}
|
|
322
|
+
return Buffer.from(await response.arrayBuffer());
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/** Headers shared by the API and asset requests (the API rejects requests without a User-Agent). */
|
|
326
|
+
function githubHeaders(token: string | undefined, accept: string): Record<string, string> {
|
|
327
|
+
const headers: Record<string, string> = {
|
|
328
|
+
Accept: accept,
|
|
329
|
+
'User-Agent': 'duckfn-docs-kit',
|
|
330
|
+
'X-GitHub-Api-Version': '2022-11-28',
|
|
331
|
+
};
|
|
332
|
+
if (token) {
|
|
333
|
+
headers.Authorization = `Bearer ${token}`;
|
|
334
|
+
}
|
|
335
|
+
return headers;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** GitHub's `digest` field (`sha256:<hex>`) as a bare lowercase hex digest; null when absent or unparseable. */
|
|
339
|
+
function digestSha256(digest: string | null | undefined): string | null {
|
|
340
|
+
if (typeof digest !== 'string') {
|
|
341
|
+
return null;
|
|
342
|
+
}
|
|
343
|
+
const [algorithm, hex] = digest.split(':');
|
|
344
|
+
if (algorithm !== 'sha256' || hex === undefined || !/^[0-9a-f]{64}$/i.test(hex)) {
|
|
345
|
+
return null;
|
|
346
|
+
}
|
|
347
|
+
return hex.toLowerCase();
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function sha256Hex(bytes: Uint8Array): string {
|
|
351
|
+
return createHash('sha256').update(bytes).digest('hex');
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
interface CachePaths {
|
|
355
|
+
file: string;
|
|
356
|
+
meta: string;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
interface CachedAsset {
|
|
360
|
+
bytes: Buffer;
|
|
361
|
+
sha256: string;
|
|
362
|
+
tag: string;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
interface CacheMeta {
|
|
366
|
+
sha256?: string;
|
|
367
|
+
tag?: string;
|
|
368
|
+
fetchedAt?: string;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function cacheFilePaths(cacheRoot: string, source: PreloadReleaseSource): CachePaths {
|
|
372
|
+
const dir = path.join(cacheRoot, source.repository.replace('/', '__'));
|
|
373
|
+
return {file: path.join(dir, source.asset), meta: path.join(dir, `${source.asset}.json`)};
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
async function readCachedAsset(paths: CachePaths): Promise<CachedAsset | null> {
|
|
377
|
+
try {
|
|
378
|
+
const [bytes, metaText] = await Promise.all([readFile(paths.file), readFile(paths.meta, 'utf8')]);
|
|
379
|
+
const meta = JSON.parse(metaText) as CacheMeta;
|
|
380
|
+
if (typeof meta.sha256 !== 'string' || !/^[0-9a-f]{64}$/.test(meta.sha256)) {
|
|
381
|
+
return null;
|
|
382
|
+
}
|
|
383
|
+
return {bytes, sha256: meta.sha256, tag: meta.tag ?? ''};
|
|
384
|
+
} catch {
|
|
385
|
+
// No (readable) cache — the normal state on the first run.
|
|
386
|
+
return null;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
async function writeCachedAsset(paths: CachePaths, bytes: Buffer, sha256: string, tag: string): Promise<void> {
|
|
391
|
+
await mkdir(path.dirname(paths.file), {recursive: true});
|
|
392
|
+
await writeFileAtomically(paths.file, bytes);
|
|
393
|
+
const meta: CacheMeta = {sha256, tag, fetchedAt: new Date().toISOString()};
|
|
394
|
+
await writeFileAtomically(paths.meta, Buffer.from(`${JSON.stringify(meta, null, 2)}\n`));
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/** Writes `bytes` to `file` unless it already holds exactly those bytes. */
|
|
398
|
+
async function writeIfChanged(
|
|
399
|
+
file: string,
|
|
400
|
+
bytes: Buffer,
|
|
401
|
+
sha256: string,
|
|
402
|
+
log: (text: string) => void,
|
|
403
|
+
label: string,
|
|
404
|
+
): Promise<void> {
|
|
405
|
+
const current = await readFile(file).catch(() => null);
|
|
406
|
+
if (current && sha256Hex(current) === sha256) {
|
|
407
|
+
log(`${label} up to date`);
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
await mkdir(path.dirname(file), {recursive: true});
|
|
411
|
+
await writeFileAtomically(file, bytes);
|
|
412
|
+
log(`wrote ${label} (${bytes.length} bytes)`);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/** Write via a temporary file + rename so a reader never sees a partial file. */
|
|
416
|
+
async function writeFileAtomically(file: string, bytes: Buffer): Promise<void> {
|
|
417
|
+
const tmp = `${file}.tmp-${process.pid.toString(36)}-${Date.now().toString(36)}`;
|
|
418
|
+
await writeFile(tmp, bytes);
|
|
419
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
420
|
+
try {
|
|
421
|
+
await rename(tmp, file);
|
|
422
|
+
return;
|
|
423
|
+
} catch (error) {
|
|
424
|
+
if (attempt >= 3) {
|
|
425
|
+
await unlink(tmp).catch(() => undefined);
|
|
426
|
+
throw error;
|
|
427
|
+
}
|
|
428
|
+
// Windows can hold a transient lock on the destination (antivirus,
|
|
429
|
+
// indexers); wait a moment and try again.
|
|
430
|
+
await new Promise((resolve) => setTimeout(resolve, 50 * attempt));
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* Prefixes site-relative paths with the deploy's baseUrl so the runtime can
|
|
437
|
+
* resolve them against the page origin; absolute URLs pass through, and the
|
|
438
|
+
* build-only `release` source is stripped from what the browser sees.
|
|
439
|
+
*/
|
|
440
|
+
function resolveForSite(entry: PreloadEntry, baseUrl: string): PreloadEntry {
|
|
441
|
+
if (typeof entry === 'string' || !('url' in entry)) {
|
|
442
|
+
return entry;
|
|
443
|
+
}
|
|
444
|
+
return isAbsoluteHttpUrl(entry.url)
|
|
445
|
+
? {url: entry.url}
|
|
446
|
+
: {url: joinBaseUrl(baseUrl, entry.url)};
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/** `/duckfn/` + `duckdb-extensions/x.wasm` -> `/duckfn/duckdb-extensions/x.wasm`. */
|
|
450
|
+
function joinBaseUrl(baseUrl: string, url: string): string {
|
|
451
|
+
const base = baseUrl.replace(/^\/+|\/+$/g, '');
|
|
452
|
+
return base === '' ? `/${url}` : `/${base}/${url}`;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
async function fileExists(file: string): Promise<boolean> {
|
|
456
|
+
try {
|
|
457
|
+
await stat(file);
|
|
458
|
+
return true;
|
|
459
|
+
} catch {
|
|
460
|
+
return false;
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
function shortHash(sha256: string): string {
|
|
465
|
+
return sha256.slice(0, 12);
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
function messageOf(error: unknown): string {
|
|
469
|
+
return error instanceof Error ? error.message : String(error);
|
|
470
|
+
}
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import type {Plugin} from 'unified';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Turns a fenced SQL block whose metastring is a JSON config with
|
|
5
|
+
* `{"type":"duckfn", …}` into a `<dfk-sql>` custom element, so the docs site
|
|
6
|
+
* can render it as a runnable example.
|
|
7
|
+
*
|
|
8
|
+
* The original `code` node is kept as the element's *child*: Docusaurus'
|
|
9
|
+
* `codeCompatPlugin` still stamps `metastring` onto it and the classic theme
|
|
10
|
+
* renders it as a regular `@theme/CodeBlock`. That child is the block's
|
|
11
|
+
* prerendered text and nothing else — `<dfk-sql>` has no default slot and hides
|
|
12
|
+
* unslotted children through CSS, because the code view is a CodeMirror editor.
|
|
13
|
+
* The SQL text and the parsed config travel as string attributes (`sql` /
|
|
14
|
+
* `config`) — React 19 reconciles string props onto custom elements as
|
|
15
|
+
* attributes, so they survive prerendering and hydration.
|
|
16
|
+
*
|
|
17
|
+
* The element also gets a prerendered *placeholder* (see {@link skeleton}): one
|
|
18
|
+
* bar per line of the SQL, which is what the reader sees before this package's
|
|
19
|
+
* JS arrives and `<dfk-sql>` upgrades.
|
|
20
|
+
*
|
|
21
|
+
* Unlike Docusaurus' own `key=value` metastring format, the config here is
|
|
22
|
+
* JSON, which allows nested fields (`option: {…}`) for future renderers.
|
|
23
|
+
*
|
|
24
|
+
* This is Node-side build code: it must not touch `window` / `document`, and it
|
|
25
|
+
* must not import any browser module (type-only imports are fine).
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
/** The JSON payload written after the info string of a runnable SQL block. */
|
|
29
|
+
export interface RunnableSqlConfig {
|
|
30
|
+
/** Marks the block as a duckfn runnable example; the only value today. */
|
|
31
|
+
type: 'duckfn';
|
|
32
|
+
/**
|
|
33
|
+
* Which result renderer to use. Defaults to `table` at runtime, except that a
|
|
34
|
+
* single-column single-row result degrades to `text` (a bare scalar reads
|
|
35
|
+
* better as a line than as a 1×1 table).
|
|
36
|
+
*
|
|
37
|
+
* `html` and `iframe` are the same renderer: both sandbox the markup in an
|
|
38
|
+
* iframe, so scripts run with an opaque origin.
|
|
39
|
+
*/
|
|
40
|
+
show?: 'table' | 'html' | 'iframe' | 'svg' | 'text';
|
|
41
|
+
/**
|
|
42
|
+
* The column holding the markup, for the preview renderers. A single-column
|
|
43
|
+
* result is unambiguous and is used as-is.
|
|
44
|
+
*/
|
|
45
|
+
field?: string;
|
|
46
|
+
/** The column to label each preview tab with; falls back to `Row N`. */
|
|
47
|
+
tab_name?: string;
|
|
48
|
+
/** Presentation knobs for the preview renderers; see `option.width` etc. */
|
|
49
|
+
option?: {
|
|
50
|
+
/** CSS length for the preview box (e.g. `'100%'`, `'640px'`). */
|
|
51
|
+
width?: string;
|
|
52
|
+
height?: string;
|
|
53
|
+
/**
|
|
54
|
+
* `sandbox` tokens for the `iframe` renderer, replacing the default
|
|
55
|
+
* `allow-scripts`. Only set this to *widen* what the report may do — the
|
|
56
|
+
* default deliberately omits `allow-same-origin`.
|
|
57
|
+
*/
|
|
58
|
+
sandbox?: string;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* duckfn community extensions to `LOAD` before running the block. The kit
|
|
62
|
+
* never hard-codes an extension name; the docs source names what it needs.
|
|
63
|
+
*/
|
|
64
|
+
extensions?: string[];
|
|
65
|
+
/** A repository serving the extensions, instead of the DuckDB default. */
|
|
66
|
+
repository?: string;
|
|
67
|
+
/**
|
|
68
|
+
* Allows `LOAD` to accept extensions without a valid signature. Opt-in
|
|
69
|
+
* per block, and only meaningful for the *first* block that initialises the
|
|
70
|
+
* shared runtime — `open()` fixes it for the instance.
|
|
71
|
+
*/
|
|
72
|
+
allowUnsignedExtensions?: boolean;
|
|
73
|
+
/**
|
|
74
|
+
* Forward-compatible fields: the remark plugin passes the whole object
|
|
75
|
+
* through untouched, so a newer kit version can read new keys without the
|
|
76
|
+
* docs source changing.
|
|
77
|
+
*/
|
|
78
|
+
[key: string]: unknown;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface RunnableSqlOptions {
|
|
82
|
+
/**
|
|
83
|
+
* Reserved for future remark-level options (kept so sites passing an empty
|
|
84
|
+
* options object keep typechecking). Site-wide extension preloading is
|
|
85
|
+
* configured on the `dfkExtensions` plugin (`sql/extensions`), not here.
|
|
86
|
+
*/
|
|
87
|
+
[key: string]: unknown;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** The custom element the plugin emits; must match `register.ts`. */
|
|
91
|
+
export const DFK_SQL_TAG = 'dfk-sql';
|
|
92
|
+
|
|
93
|
+
interface CodeNode {
|
|
94
|
+
type: string;
|
|
95
|
+
lang?: string | null;
|
|
96
|
+
meta?: string | null;
|
|
97
|
+
value?: unknown;
|
|
98
|
+
children?: unknown[];
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
interface ParentNode {
|
|
102
|
+
children?: unknown[];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Parse the metastring; `null` means "not a runnable block, leave it alone". */
|
|
106
|
+
function parseConfig(meta: string | null | undefined): RunnableSqlConfig | null {
|
|
107
|
+
if (!meta) {
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
try {
|
|
111
|
+
const parsed: unknown = JSON.parse(meta);
|
|
112
|
+
if (
|
|
113
|
+
typeof parsed === 'object' &&
|
|
114
|
+
parsed !== null &&
|
|
115
|
+
(parsed as {type?: unknown}).type === 'duckfn'
|
|
116
|
+
) {
|
|
117
|
+
return parsed as RunnableSqlConfig;
|
|
118
|
+
}
|
|
119
|
+
} catch {
|
|
120
|
+
// Not JSON (a plain `sql` block, or Docusaurus' `key=value` metastrings):
|
|
121
|
+
// a normal code block, nothing to do.
|
|
122
|
+
}
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* How many bars the placeholder draws: the SQL's own line count, floored at one
|
|
128
|
+
* (an empty fence still needs a row).
|
|
129
|
+
*
|
|
130
|
+
* One line is exactly one CodeMirror line box, so a placeholder of this shape
|
|
131
|
+
* leaves the block as tall as the editor that eventually replaces it. The
|
|
132
|
+
* component counts the same way before it builds its own (shadow-tree) copy of
|
|
133
|
+
* the placeholder — the duplication is deliberate: this is Node build code and
|
|
134
|
+
* must not be imported by anything the browser bundles.
|
|
135
|
+
*/
|
|
136
|
+
function sqlLineCount(sql: string): number {
|
|
137
|
+
return Math.max(1, sql.split('\n').length);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* The placeholder shown before the element upgrades: one bar per SQL line.
|
|
142
|
+
*
|
|
143
|
+
* Until this package's JS runs there is no shadow tree, and `sql.css` hides
|
|
144
|
+
* every unslotted child of a `<dfk-sql>` — the code node kept below included —
|
|
145
|
+
* so without this the block would be an invisible hole that pops in and pushes
|
|
146
|
+
* the rest of the page down. The bars are real children, which also means no JS
|
|
147
|
+
* has to remove them: once the element upgrades they are simply not slotted.
|
|
148
|
+
*
|
|
149
|
+
* `className` rather than `class`: MDX compiles this to a React element, and
|
|
150
|
+
* React wants the DOM prop spelling on built-in tags.
|
|
151
|
+
*/
|
|
152
|
+
function skeleton(sql: string): Record<string, unknown> {
|
|
153
|
+
return {
|
|
154
|
+
type: 'mdxJsxFlowElement',
|
|
155
|
+
name: 'div',
|
|
156
|
+
attributes: [{type: 'mdxJsxAttribute', name: 'className', value: 'dfk-sql-editor-skeleton'}],
|
|
157
|
+
children: Array.from({length: sqlLineCount(sql)}, () => ({
|
|
158
|
+
type: 'mdxJsxFlowElement',
|
|
159
|
+
name: 'span',
|
|
160
|
+
attributes: [
|
|
161
|
+
{type: 'mdxJsxAttribute', name: 'className', value: 'dfk-sql-editor-skeleton-line'},
|
|
162
|
+
],
|
|
163
|
+
children: [],
|
|
164
|
+
})),
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function wrapRunnableSql(code: CodeNode, config: RunnableSqlConfig): Record<string, unknown> {
|
|
169
|
+
const sql = String(code.value ?? '');
|
|
170
|
+
return {
|
|
171
|
+
type: 'mdxJsxFlowElement',
|
|
172
|
+
name: DFK_SQL_TAG,
|
|
173
|
+
attributes: [
|
|
174
|
+
{type: 'mdxJsxAttribute', name: 'config', value: JSON.stringify(config)},
|
|
175
|
+
{type: 'mdxJsxAttribute', name: 'sql', value: sql},
|
|
176
|
+
],
|
|
177
|
+
// The placeholder first, then the code node. Both are unslotted, so once the
|
|
178
|
+
// element upgrades neither renders: the placeholder has done its job by then
|
|
179
|
+
// (`sql.css` draws it before the upgrade, and hides every *other* unslotted
|
|
180
|
+
// child) and the code node is the prerendered text the editor replaces.
|
|
181
|
+
children: [skeleton(sql), code],
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export const remarkRunnableSql: Plugin<[RunnableSqlOptions?]> =
|
|
186
|
+
(_options = {}) =>
|
|
187
|
+
(tree) => {
|
|
188
|
+
const walk = (node: unknown): void => {
|
|
189
|
+
if (typeof node !== 'object' || node === null) {
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
const parent = node as ParentNode;
|
|
193
|
+
if (!Array.isArray(parent.children)) {
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
parent.children = parent.children.map((child) => {
|
|
197
|
+
if (typeof child !== 'object' || child === null) {
|
|
198
|
+
return child;
|
|
199
|
+
}
|
|
200
|
+
const candidate = child as CodeNode;
|
|
201
|
+
if (candidate.type === 'code' && candidate.lang === 'sql') {
|
|
202
|
+
const config = parseConfig(candidate.meta);
|
|
203
|
+
if (config) {
|
|
204
|
+
return wrapRunnableSql(candidate, config);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
walk(candidate);
|
|
208
|
+
return candidate;
|
|
209
|
+
});
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
walk(tree);
|
|
213
|
+
};
|