mdorigin 0.1.8 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,47 @@
1
+ import path from 'node:path';
2
+ import { syncCloudflareR2 } from '../cloudflare.js';
3
+ export async function runSyncCloudflareR2Command(argv) {
4
+ const args = parseArgs(argv);
5
+ if (args.help) {
6
+ console.log('Usage: mdorigin sync cloudflare-r2 --dir ./dist/cloudflare --bucket <bucket-name> [--force]');
7
+ return;
8
+ }
9
+ if (!args.bucket) {
10
+ console.error('Usage: mdorigin sync cloudflare-r2 --dir ./dist/cloudflare --bucket <bucket-name> [--force]');
11
+ process.exitCode = 1;
12
+ return;
13
+ }
14
+ const result = await syncCloudflareR2({
15
+ dir: path.resolve(args.dir ?? 'dist/cloudflare'),
16
+ bucketName: args.bucket,
17
+ force: args.force,
18
+ });
19
+ console.log(`synced ${result.uploadedCount} R2 object(s), skipped ${result.skippedCount}, state written to ${result.stateFile}`);
20
+ }
21
+ function parseArgs(argv) {
22
+ const result = {};
23
+ for (let index = 0; index < argv.length; index += 1) {
24
+ const argument = argv[index];
25
+ const nextValue = argv[index + 1];
26
+ if (argument === '--help' || argument === '-h') {
27
+ result.help = true;
28
+ continue;
29
+ }
30
+ if (argument === '--dir' && nextValue) {
31
+ result.dir = nextValue;
32
+ index += 1;
33
+ continue;
34
+ }
35
+ if (argument === '--bucket' && nextValue) {
36
+ result.bucket = nextValue;
37
+ index += 1;
38
+ continue;
39
+ }
40
+ if (argument === '--force') {
41
+ result.force = true;
42
+ continue;
43
+ }
44
+ throw new Error(`Unknown argument for mdorigin sync cloudflare-r2: ${argument}`);
45
+ }
46
+ return result;
47
+ }
@@ -1,2 +1,2 @@
1
1
  export { createCloudflareWorker } from './adapters/cloudflare.js';
2
- export type { CloudflareManifest, CloudflareManifestEntry, ExportedHandlerLike, } from './adapters/cloudflare.js';
2
+ export type { CloudflareBundleRuntimeConfig, CloudflareManifest, CloudflareManifestEntry, ExportedHandlerLike, } from './adapters/cloudflare.js';
@@ -1,34 +1,72 @@
1
- import type { CloudflareManifest, CloudflareManifestEntry } from './adapters/cloudflare.js';
1
+ import type { CloudflareBundleRuntimeConfig, CloudflareManifest, CloudflareManifestEntry } from './adapters/cloudflare.js';
2
2
  import { createCloudflareWorker } from './adapters/cloudflare.js';
3
3
  import type { ResolvedSiteConfig } from './core/site-config.js';
4
4
  export { createCloudflareWorker };
5
- export type { CloudflareManifest, CloudflareManifestEntry };
5
+ export type { CloudflareBundleRuntimeConfig, CloudflareManifest, CloudflareManifestEntry, };
6
+ export type CloudflareBinaryMode = 'inline' | 'external';
6
7
  export interface BuildCloudflareManifestOptions {
7
8
  rootDir: string;
8
9
  siteConfig: ResolvedSiteConfig;
9
10
  searchDir?: string;
11
+ binaryMode?: CloudflareBinaryMode;
12
+ assetsMaxBytes?: number;
13
+ r2Binding?: string;
10
14
  }
11
- export interface WriteCloudflareBundleOptions {
12
- rootDir: string;
15
+ export interface WriteCloudflareBundleOptions extends BuildCloudflareManifestOptions {
13
16
  outDir: string;
14
- siteConfig: ResolvedSiteConfig;
15
17
  packageImport?: string;
16
- searchDir?: string;
17
18
  configModulePath?: string;
18
19
  }
20
+ export interface CloudflareBundleMetadata {
21
+ version: 2;
22
+ workerEntry: string;
23
+ binaryMode: CloudflareBinaryMode;
24
+ assetsMaxBytes?: number;
25
+ assetsDir?: string;
26
+ r2Dir?: string;
27
+ r2Binding?: string;
28
+ siteTitle?: string;
29
+ stagedObjects: Array<{
30
+ kind: 'binary' | 'search';
31
+ path: string;
32
+ mediaType: string;
33
+ storageKind: 'assets' | 'r2';
34
+ storageKey: string;
35
+ file: string;
36
+ byteSize: number;
37
+ }>;
38
+ }
19
39
  export interface InitCloudflareProjectOptions {
20
40
  projectDir: string;
21
41
  workerEntry: string;
22
42
  workerName?: string;
23
43
  siteTitle?: string;
24
44
  compatibilityDate?: string;
45
+ r2Bucket?: string;
46
+ force?: boolean;
47
+ }
48
+ export interface SyncCloudflareR2Options {
49
+ dir: string;
50
+ bucketName: string;
25
51
  force?: boolean;
52
+ runCommand?: typeof runWranglerCommand;
53
+ }
54
+ export interface SyncCloudflareR2Result {
55
+ uploadedCount: number;
56
+ skippedCount: number;
57
+ stateFile: string;
26
58
  }
27
59
  export declare function buildCloudflareManifest(options: BuildCloudflareManifestOptions): Promise<CloudflareManifest>;
28
60
  export declare function writeCloudflareBundle(options: WriteCloudflareBundleOptions): Promise<{
29
61
  manifest: CloudflareManifest;
30
62
  workerFile: string;
63
+ bundleFile: string;
31
64
  }>;
32
65
  export declare function initCloudflareProject(options: InitCloudflareProjectOptions): Promise<{
33
66
  configFile: string;
34
67
  }>;
68
+ export declare function syncCloudflareR2(options: SyncCloudflareR2Options): Promise<SyncCloudflareR2Result>;
69
+ declare function runWranglerCommand(command: string, args: string[]): {
70
+ status: number | null;
71
+ stderr: string;
72
+ };
@@ -1,12 +1,24 @@
1
- import { mkdir, readdir, readFile, realpath, stat, writeFile } from 'node:fs/promises';
1
+ import { spawnSync } from 'node:child_process';
2
+ import { createHash } from 'node:crypto';
3
+ import { createReadStream } from 'node:fs';
4
+ import { copyFile, mkdir, readFile, readdir, realpath, rm, stat, writeFile, } from 'node:fs/promises';
2
5
  import path from 'node:path';
3
6
  import { createCloudflareWorker } from './adapters/cloudflare.js';
4
- import { getMediaTypeForPath, isLikelyTextPath, normalizeContentPath, } from './core/content-store.js';
7
+ import { getMediaTypeForPath, isIgnoredContentName, isLikelyTextPath, normalizeContentPath, } from './core/content-store.js';
5
8
  export { createCloudflareWorker };
9
+ const DEFAULT_ASSETS_MAX_BYTES = 25 * 1024 * 1024;
10
+ const DEFAULT_ASSETS_BINDING = 'ASSETS';
11
+ const DEFAULT_R2_BINDING = 'MDORIGIN_R2';
12
+ const BUNDLE_FILE_NAME = 'bundle.json';
13
+ const R2_STATE_FILE_NAME = 'r2-sync-state.json';
14
+ const SEARCH_ASSETS_PREFIX = '__mdorigin/search';
6
15
  export async function buildCloudflareManifest(options) {
7
16
  const rootDir = path.resolve(options.rootDir);
8
17
  const files = await listFiles(rootDir);
9
18
  const entries = [];
19
+ const binaryMode = options.binaryMode ?? 'inline';
20
+ const assetsMaxBytes = options.assetsMaxBytes ?? DEFAULT_ASSETS_MAX_BYTES;
21
+ const r2Binding = options.r2Binding ?? DEFAULT_R2_BINDING;
10
22
  for (const filePath of files) {
11
23
  const relativePath = path.relative(rootDir, filePath).replaceAll(path.sep, '/');
12
24
  const normalizedPath = normalizeContentPath(relativePath);
@@ -23,32 +35,56 @@ export async function buildCloudflareManifest(options) {
23
35
  });
24
36
  continue;
25
37
  }
26
- entries.push({
27
- path: normalizedPath,
28
- kind: 'binary',
29
- mediaType,
30
- base64: (await readFile(filePath)).toString('base64'),
31
- });
38
+ if (binaryMode === 'inline') {
39
+ const bytes = await readFile(filePath);
40
+ entries.push({
41
+ path: normalizedPath,
42
+ kind: 'binary',
43
+ mediaType,
44
+ base64: bytes.toString('base64'),
45
+ });
46
+ continue;
47
+ }
48
+ const fileStats = await stat(filePath);
49
+ entries.push(await buildExternalBinaryEntry(filePath, normalizedPath, mediaType, fileStats.size, {
50
+ assetsMaxBytes,
51
+ r2Binding,
52
+ }));
32
53
  }
33
- const searchEntries = options.searchDir
34
- ? await readBundleEntries(path.resolve(options.searchDir))
54
+ const externalSearchEntries = options.searchDir
55
+ ? await buildSearchBundleEntries(path.resolve(options.searchDir), assetsMaxBytes)
35
56
  : undefined;
36
57
  entries.sort((left, right) => left.path.localeCompare(right.path));
37
58
  return {
38
59
  entries,
39
60
  siteConfig: options.siteConfig,
40
- searchEntries,
61
+ externalSearchEntries,
62
+ runtime: binaryMode === 'external' || (externalSearchEntries?.length ?? 0) > 0
63
+ ? {
64
+ binaryMode,
65
+ r2Binding,
66
+ }
67
+ : {
68
+ binaryMode,
69
+ },
41
70
  };
42
71
  }
43
72
  export async function writeCloudflareBundle(options) {
44
73
  const outDir = path.resolve(options.outDir);
74
+ const binaryMode = options.binaryMode ?? 'inline';
75
+ const assetsMaxBytes = options.assetsMaxBytes ?? DEFAULT_ASSETS_MAX_BYTES;
76
+ const r2Binding = options.r2Binding ?? DEFAULT_R2_BINDING;
45
77
  const manifest = await buildCloudflareManifest({
46
78
  rootDir: options.rootDir,
47
79
  siteConfig: options.siteConfig,
48
80
  searchDir: options.searchDir,
81
+ binaryMode,
82
+ assetsMaxBytes,
83
+ r2Binding,
49
84
  });
50
85
  const packageImport = options.packageImport ?? 'mdorigin/cloudflare-runtime';
51
86
  const workerFile = path.join(outDir, 'worker.mjs');
87
+ const bundleFile = path.join(outDir, BUNDLE_FILE_NAME);
52
88
  const configImportPath = options.configModulePath
53
89
  ? toPosixPath(path.relative(outDir, options.configModulePath))
54
90
  : null;
@@ -83,10 +119,18 @@ export async function writeCloudflareBundle(options) {
83
119
  '',
84
120
  ].join('\n');
85
121
  await mkdir(outDir, { recursive: true });
122
+ const metadata = await writeExternalStaging(path.resolve(options.rootDir), options.searchDir ? path.resolve(options.searchDir) : undefined, outDir, manifest, {
123
+ binaryMode,
124
+ assetsMaxBytes,
125
+ r2Binding,
126
+ siteTitle: options.siteConfig.siteTitle,
127
+ });
86
128
  await writeFile(workerFile, workerSource, 'utf8');
129
+ await writeFile(bundleFile, JSON.stringify(metadata, null, 2), 'utf8');
87
130
  return {
88
131
  manifest,
89
132
  workerFile,
133
+ bundleFile,
90
134
  };
91
135
  }
92
136
  export async function initCloudflareProject(options) {
@@ -96,8 +140,14 @@ export async function initCloudflareProject(options) {
96
140
  if (existing && !options.force) {
97
141
  throw new Error(`Refusing to overwrite ${configFile}. Re-run with --force to replace it.`);
98
142
  }
143
+ const bundleMetadata = await readCloudflareBundleMetadata(options.workerEntry);
144
+ if (bundleMetadata &&
145
+ bundleMetadata.stagedObjects.some((object) => object.storageKind === 'r2') &&
146
+ !options.r2Bucket) {
147
+ throw new Error('Cloudflare bundle contains R2-backed staged objects. Re-run init cloudflare with --r2-bucket <bucket-name>.');
148
+ }
99
149
  const workerName = options.workerName ??
100
- slugifyWorkerName(options.siteTitle) ??
150
+ slugifyWorkerName(bundleMetadata?.siteTitle ?? options.siteTitle) ??
101
151
  'mdorigin-site';
102
152
  const compatibilityDate = options.compatibilityDate ?? '2026-03-20';
103
153
  const wranglerConfig = [
@@ -107,13 +157,273 @@ export async function initCloudflareProject(options) {
107
157
  ` "main": ${JSON.stringify(toPosixPath(path.relative(projectDir, options.workerEntry)))},`,
108
158
  ` "compatibility_date": ${JSON.stringify(compatibilityDate)},`,
109
159
  ' "compatibility_flags": ["nodejs_compat"]',
160
+ bundleMetadata?.assetsDir
161
+ ? [
162
+ ',',
163
+ ' "assets": {',
164
+ ` "directory": ${JSON.stringify(toPosixPath(path.relative(projectDir, path.join(path.dirname(options.workerEntry), bundleMetadata.assetsDir))))},`,
165
+ ` "binding": ${JSON.stringify(DEFAULT_ASSETS_BINDING)},`,
166
+ ' "run_worker_first": true',
167
+ ' }',
168
+ ].join('\n')
169
+ : '',
170
+ bundleMetadata &&
171
+ bundleMetadata.stagedObjects.some((object) => object.storageKind === 'r2') &&
172
+ bundleMetadata.r2Binding &&
173
+ options.r2Bucket
174
+ ? [
175
+ ',',
176
+ ' "r2_buckets": [',
177
+ ' {',
178
+ ` "binding": ${JSON.stringify(bundleMetadata.r2Binding)},`,
179
+ ` "bucket_name": ${JSON.stringify(options.r2Bucket)}`,
180
+ ' }',
181
+ ' ]',
182
+ ].join('\n')
183
+ : '',
110
184
  '}',
111
185
  '',
112
- ].join('\n');
186
+ ]
187
+ .filter(Boolean)
188
+ .join('\n');
113
189
  await mkdir(projectDir, { recursive: true });
114
190
  await writeFile(configFile, wranglerConfig, 'utf8');
115
191
  return { configFile };
116
192
  }
193
+ export async function syncCloudflareR2(options) {
194
+ const outDir = path.resolve(options.dir);
195
+ const bundleFile = path.join(outDir, BUNDLE_FILE_NAME);
196
+ const metadata = await readBundleMetadataFile(bundleFile);
197
+ const r2Objects = metadata.stagedObjects.filter((object) => object.storageKind === 'r2');
198
+ if (r2Objects.length === 0) {
199
+ throw new Error(`No R2-backed staged objects found in ${bundleFile}.`);
200
+ }
201
+ const stateFile = path.join(outDir, R2_STATE_FILE_NAME);
202
+ const state = await readR2SyncState(stateFile);
203
+ const runCommand = options.runCommand ?? runWranglerCommand;
204
+ let uploadedCount = 0;
205
+ let skippedCount = 0;
206
+ for (const object of r2Objects) {
207
+ const stateKey = `${options.bucketName}:${object.storageKey}`;
208
+ if (!options.force && state.uploaded[stateKey]) {
209
+ skippedCount += 1;
210
+ continue;
211
+ }
212
+ const filePath = path.join(outDir, object.file);
213
+ const result = runCommand('wrangler', [
214
+ 'r2',
215
+ 'object',
216
+ 'put',
217
+ `${options.bucketName}/${object.storageKey}`,
218
+ '--file',
219
+ filePath,
220
+ '--content-type',
221
+ object.mediaType,
222
+ '--remote',
223
+ ]);
224
+ if (result.status !== 0) {
225
+ throw new Error(result.stderr || `Failed to upload R2 object ${object.storageKey}.`);
226
+ }
227
+ state.uploaded[stateKey] = {
228
+ syncedAt: new Date().toISOString(),
229
+ };
230
+ uploadedCount += 1;
231
+ }
232
+ await writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8');
233
+ return {
234
+ uploadedCount,
235
+ skippedCount,
236
+ stateFile,
237
+ };
238
+ }
239
+ async function buildExternalBinaryEntry(filePath, normalizedPath, mediaType, byteSize, options) {
240
+ if (byteSize <= options.assetsMaxBytes) {
241
+ return {
242
+ path: normalizedPath,
243
+ kind: 'binary',
244
+ mediaType,
245
+ storageKind: 'assets',
246
+ storageKey: normalizedPath,
247
+ byteSize,
248
+ };
249
+ }
250
+ return {
251
+ path: normalizedPath,
252
+ kind: 'binary',
253
+ mediaType,
254
+ storageKind: 'r2',
255
+ storageKey: await buildR2StorageKey(filePath, normalizedPath),
256
+ byteSize,
257
+ };
258
+ }
259
+ async function writeExternalStaging(rootDir, searchDir, outDir, manifest, options) {
260
+ const assetsDir = path.join(outDir, 'assets');
261
+ const r2Dir = path.join(outDir, 'r2');
262
+ await rm(assetsDir, { recursive: true, force: true });
263
+ await rm(r2Dir, { recursive: true, force: true });
264
+ const stagedObjects = new Map();
265
+ let hasAssets = false;
266
+ let hasR2 = false;
267
+ if (options.binaryMode === 'external') {
268
+ for (const entry of manifest.entries) {
269
+ if (entry.kind !== 'binary' || !('storageKind' in entry)) {
270
+ continue;
271
+ }
272
+ const sourceFile = path.resolve(rootDir, entry.path);
273
+ if (entry.storageKind === 'assets') {
274
+ const targetFile = path.join(assetsDir, entry.storageKey);
275
+ await mkdir(path.dirname(targetFile), { recursive: true });
276
+ await copyFile(sourceFile, targetFile);
277
+ hasAssets = true;
278
+ stagedObjects.set(`assets:${entry.storageKey}`, {
279
+ kind: 'binary',
280
+ path: entry.path,
281
+ mediaType: entry.mediaType,
282
+ storageKind: 'assets',
283
+ storageKey: entry.storageKey,
284
+ file: toPosixPath(path.join('assets', entry.storageKey)),
285
+ byteSize: entry.byteSize,
286
+ });
287
+ continue;
288
+ }
289
+ const relativeFile = toPosixPath(path.join('r2', entry.storageKey));
290
+ const targetFile = path.join(outDir, relativeFile);
291
+ if (!stagedObjects.has(`r2:${entry.storageKey}`)) {
292
+ await mkdir(path.dirname(targetFile), { recursive: true });
293
+ await copyFile(sourceFile, targetFile);
294
+ hasR2 = true;
295
+ stagedObjects.set(`r2:${entry.storageKey}`, {
296
+ kind: 'binary',
297
+ path: entry.path,
298
+ mediaType: entry.mediaType,
299
+ storageKind: 'r2',
300
+ storageKey: entry.storageKey,
301
+ file: relativeFile,
302
+ byteSize: entry.byteSize,
303
+ });
304
+ }
305
+ }
306
+ }
307
+ if (searchDir && manifest.externalSearchEntries) {
308
+ for (const entry of manifest.externalSearchEntries) {
309
+ const sourceFile = path.join(searchDir, entry.path);
310
+ if (entry.storageKind === 'assets') {
311
+ const targetFile = path.join(assetsDir, entry.storageKey);
312
+ await mkdir(path.dirname(targetFile), { recursive: true });
313
+ await copyFile(sourceFile, targetFile);
314
+ hasAssets = true;
315
+ stagedObjects.set(`assets:${entry.storageKey}`, {
316
+ kind: 'search',
317
+ path: entry.path,
318
+ mediaType: entry.mediaType,
319
+ storageKind: 'assets',
320
+ storageKey: entry.storageKey,
321
+ file: toPosixPath(path.join('assets', entry.storageKey)),
322
+ byteSize: entry.byteSize,
323
+ });
324
+ continue;
325
+ }
326
+ const relativeFile = toPosixPath(path.join('r2', entry.storageKey));
327
+ const targetFile = path.join(outDir, relativeFile);
328
+ if (!stagedObjects.has(`r2:${entry.storageKey}`)) {
329
+ await mkdir(path.dirname(targetFile), { recursive: true });
330
+ await copyFile(sourceFile, targetFile);
331
+ hasR2 = true;
332
+ stagedObjects.set(`r2:${entry.storageKey}`, {
333
+ kind: 'search',
334
+ path: entry.path,
335
+ mediaType: entry.mediaType,
336
+ storageKind: 'r2',
337
+ storageKey: entry.storageKey,
338
+ file: relativeFile,
339
+ byteSize: entry.byteSize,
340
+ });
341
+ }
342
+ }
343
+ }
344
+ return {
345
+ version: 2,
346
+ workerEntry: 'worker.mjs',
347
+ binaryMode: options.binaryMode,
348
+ assetsMaxBytes: options.binaryMode === 'external' || searchDir ? options.assetsMaxBytes : undefined,
349
+ assetsDir: hasAssets ? 'assets' : undefined,
350
+ r2Dir: hasR2 ? 'r2' : undefined,
351
+ r2Binding: hasR2 ? options.r2Binding : undefined,
352
+ siteTitle: options.siteTitle,
353
+ stagedObjects: Array.from(stagedObjects.values()).sort((left, right) => left.storageKey.localeCompare(right.storageKey)),
354
+ };
355
+ }
356
+ async function readCloudflareBundleMetadata(workerEntry) {
357
+ const bundleFile = path.join(path.dirname(workerEntry), BUNDLE_FILE_NAME);
358
+ if (!(await pathExists(bundleFile))) {
359
+ return null;
360
+ }
361
+ return readBundleMetadataFile(bundleFile);
362
+ }
363
+ async function readBundleMetadataFile(bundleFile) {
364
+ const parsed = JSON.parse(await readFile(bundleFile, 'utf8'));
365
+ if ('stagedObjects' in parsed && Array.isArray(parsed.stagedObjects)) {
366
+ return parsed;
367
+ }
368
+ if ('r2Objects' in parsed && Array.isArray(parsed.r2Objects)) {
369
+ return {
370
+ version: 2,
371
+ workerEntry: parsed.workerEntry,
372
+ binaryMode: parsed.binaryMode,
373
+ assetsMaxBytes: parsed.assetsMaxBytes,
374
+ assetsDir: parsed.assetsDir,
375
+ r2Dir: parsed.r2Dir,
376
+ r2Binding: parsed.r2Binding,
377
+ siteTitle: parsed.siteTitle,
378
+ stagedObjects: parsed.r2Objects.map((object) => ({
379
+ kind: 'binary',
380
+ path: object.path,
381
+ mediaType: object.mediaType,
382
+ storageKind: 'r2',
383
+ storageKey: object.storageKey,
384
+ file: object.file,
385
+ byteSize: object.byteSize,
386
+ })),
387
+ };
388
+ }
389
+ throw new Error(`Bundle metadata in ${bundleFile} is not supported. Rebuild the Cloudflare bundle with the current mdorigin CLI.`);
390
+ }
391
+ async function readR2SyncState(stateFile) {
392
+ if (!(await pathExists(stateFile))) {
393
+ return { version: 1, uploaded: {} };
394
+ }
395
+ return JSON.parse(await readFile(stateFile, 'utf8'));
396
+ }
397
+ async function buildR2StorageKey(filePath, normalizedPath) {
398
+ const extension = path.posix.extname(normalizedPath).toLowerCase();
399
+ const hash = await hashFile(filePath);
400
+ return extension ? `binary/${hash}${extension}` : `binary/${hash}`;
401
+ }
402
+ async function hashFile(filePath) {
403
+ const hash = createHash('sha256');
404
+ await new Promise((resolve, reject) => {
405
+ const stream = createReadStream(filePath);
406
+ stream.on('data', (chunk) => {
407
+ hash.update(chunk);
408
+ });
409
+ stream.on('end', () => resolve());
410
+ stream.on('error', reject);
411
+ });
412
+ return hash.digest('hex');
413
+ }
414
+ function runWranglerCommand(command, args) {
415
+ const result = spawnSync(command, args, {
416
+ encoding: 'utf8',
417
+ stdio: ['ignore', 'pipe', 'pipe'],
418
+ });
419
+ if (result.error) {
420
+ throw result.error;
421
+ }
422
+ return {
423
+ status: result.status,
424
+ stderr: result.stderr ?? '',
425
+ };
426
+ }
117
427
  async function listFiles(directory, visitedRealDirectories = new Set()) {
118
428
  const directoryRealPath = await realpath(directory);
119
429
  if (visitedRealDirectories.has(directoryRealPath)) {
@@ -123,6 +433,9 @@ async function listFiles(directory, visitedRealDirectories = new Set()) {
123
433
  const entries = await readdir(directory, { withFileTypes: true });
124
434
  const files = [];
125
435
  for (const entry of entries) {
436
+ if (isIgnoredContentName(entry.name)) {
437
+ continue;
438
+ }
126
439
  const fullPath = path.join(directory, entry.name);
127
440
  const entryStats = await stat(fullPath);
128
441
  if (entryStats.isDirectory()) {
@@ -135,31 +448,39 @@ async function listFiles(directory, visitedRealDirectories = new Set()) {
135
448
  }
136
449
  return files;
137
450
  }
138
- async function readBundleEntries(directory) {
451
+ async function buildSearchBundleEntries(directory, assetsMaxBytes) {
139
452
  const files = await listFiles(directory);
140
453
  const entries = [];
141
454
  for (const filePath of files) {
142
455
  const relativePath = path.relative(directory, filePath).replaceAll(path.sep, '/');
143
456
  const mediaType = getMediaTypeForPath(relativePath);
144
- if (isLikelyTextPath(relativePath) || relativePath.endsWith('.json')) {
457
+ const fileStats = await stat(filePath);
458
+ if (fileStats.size <= assetsMaxBytes) {
145
459
  entries.push({
146
460
  path: relativePath,
147
- kind: 'text',
148
461
  mediaType,
149
- text: await readFile(filePath, 'utf8'),
462
+ storageKind: 'assets',
463
+ storageKey: `${SEARCH_ASSETS_PREFIX}/${relativePath}`,
464
+ byteSize: fileStats.size,
150
465
  });
151
466
  continue;
152
467
  }
153
468
  entries.push({
154
469
  path: relativePath,
155
- kind: 'binary',
156
470
  mediaType,
157
- base64: (await readFile(filePath)).toString('base64'),
471
+ storageKind: 'r2',
472
+ storageKey: await buildSearchStorageKey(filePath, relativePath),
473
+ byteSize: fileStats.size,
158
474
  });
159
475
  }
160
476
  entries.sort((left, right) => left.path.localeCompare(right.path));
161
477
  return entries;
162
478
  }
479
+ async function buildSearchStorageKey(filePath, relativePath) {
480
+ const extension = path.posix.extname(relativePath).toLowerCase();
481
+ const hash = await hashFile(filePath);
482
+ return extension ? `search/${hash}${extension}` : `search/${hash}`;
483
+ }
163
484
  async function pathExists(filePath) {
164
485
  try {
165
486
  await stat(filePath);
package/dist/core/api.js CHANGED
@@ -13,10 +13,15 @@ export async function handleApiRoute(pathname, searchParams, options) {
13
13
  });
14
14
  }
15
15
  const topK = normalizePositiveInteger(searchParams?.get('topK')) ?? 10;
16
- const hits = await options.searchApi.search(query, { topK });
16
+ const metadata = readSearchMetadataFilters(searchParams);
17
+ const hits = await options.searchApi.search(query, {
18
+ topK,
19
+ metadata,
20
+ });
17
21
  return json(200, {
18
22
  query,
19
23
  topK,
24
+ metadata,
20
25
  count: hits.length,
21
26
  hits: hits.map(serializeSearchHit),
22
27
  });
@@ -56,6 +61,13 @@ function buildOpenApiDocument(options) {
56
61
  schema: { type: 'integer', minimum: 1, default: 10 },
57
62
  description: 'Maximum number of hits to return.',
58
63
  },
64
+ {
65
+ name: 'meta.<field>',
66
+ in: 'query',
67
+ required: false,
68
+ schema: { type: 'string' },
69
+ description: 'Exact-match metadata filter. Use query parameters such as meta.type=post or meta.section=guides.',
70
+ },
59
71
  ],
60
72
  responses: {
61
73
  '200': {
@@ -68,6 +80,10 @@ function buildOpenApiDocument(options) {
68
80
  properties: {
69
81
  query: { type: 'string' },
70
82
  topK: { type: 'integer' },
83
+ metadata: {
84
+ type: 'object',
85
+ additionalProperties: { type: 'string' },
86
+ },
71
87
  count: { type: 'integer' },
72
88
  hits: {
73
89
  type: 'array',
@@ -130,6 +146,23 @@ function buildOpenApiDocument(options) {
130
146
  },
131
147
  };
132
148
  }
149
+ function readSearchMetadataFilters(searchParams) {
150
+ if (!searchParams) {
151
+ return undefined;
152
+ }
153
+ const metadata = {};
154
+ for (const [key, value] of searchParams.entries()) {
155
+ if (!key.startsWith('meta.') || value.trim() === '') {
156
+ continue;
157
+ }
158
+ const metadataKey = key.slice('meta.'.length).trim();
159
+ if (metadataKey === '') {
160
+ continue;
161
+ }
162
+ metadata[metadataKey] = value;
163
+ }
164
+ return Object.keys(metadata).length > 0 ? metadata : undefined;
165
+ }
133
166
  function serializeSearchHit(hit) {
134
167
  return {
135
168
  docId: hit.docId,
@@ -15,6 +15,7 @@ export interface ContentStore {
15
15
  get(contentPath: string): Promise<ContentEntry | null>;
16
16
  listDirectory(contentPath: string): Promise<ContentDirectoryEntry[] | null>;
17
17
  }
18
+ export declare function isIgnoredContentName(name: string): boolean;
18
19
  export declare class MemoryContentStore implements ContentStore {
19
20
  private readonly entries;
20
21
  constructor(entries: Iterable<ContentEntry>);