mdorigin 0.1.8 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/adapters/cloudflare.d.ts +42 -5
- package/dist/adapters/cloudflare.js +250 -10
- package/dist/adapters/node.js +2 -2
- package/dist/cli/build-cloudflare.js +27 -2
- package/dist/cli/build-search.js +15 -4
- package/dist/cli/help.js +8 -7
- package/dist/cli/init-cloudflare.js +7 -1
- package/dist/cli/main.js +5 -0
- package/dist/cli/search.js +16 -4
- package/dist/cli/sync-cloudflare-r2.d.ts +1 -0
- package/dist/cli/sync-cloudflare-r2.js +47 -0
- package/dist/cloudflare-runtime.d.ts +1 -1
- package/dist/cloudflare.d.ts +44 -6
- package/dist/cloudflare.js +340 -19
- package/dist/core/api.js +34 -1
- package/dist/core/content-store.d.ts +1 -0
- package/dist/core/content-store.js +8 -1
- package/dist/core/extensions.d.ts +5 -9
- package/dist/core/markdown.d.ts +1 -0
- package/dist/core/markdown.js +9 -2
- package/dist/core/request-handler.js +46 -60
- package/dist/core/site-config.d.ts +4 -10
- package/dist/core/site-config.js +18 -10
- package/dist/html/template.d.ts +5 -9
- package/dist/html/template.js +25 -25
- package/dist/html/theme.d.ts +1 -2
- package/dist/html/theme.js +2 -642
- package/dist/index-builder.js +5 -4
- package/dist/search.d.ts +27 -4
- package/dist/search.js +301 -38
- package/package.json +5 -3
- package/dist/html/template-kind.d.ts +0 -1
- package/dist/html/template-kind.js +0 -1
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { syncCloudflareR2 } from '../cloudflare.js';
|
|
3
|
+
export async function runSyncCloudflareR2Command(argv) {
|
|
4
|
+
const args = parseArgs(argv);
|
|
5
|
+
if (args.help) {
|
|
6
|
+
console.log('Usage: mdorigin sync cloudflare-r2 --dir ./dist/cloudflare --bucket <bucket-name> [--force]');
|
|
7
|
+
return;
|
|
8
|
+
}
|
|
9
|
+
if (!args.bucket) {
|
|
10
|
+
console.error('Usage: mdorigin sync cloudflare-r2 --dir ./dist/cloudflare --bucket <bucket-name> [--force]');
|
|
11
|
+
process.exitCode = 1;
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
14
|
+
const result = await syncCloudflareR2({
|
|
15
|
+
dir: path.resolve(args.dir ?? 'dist/cloudflare'),
|
|
16
|
+
bucketName: args.bucket,
|
|
17
|
+
force: args.force,
|
|
18
|
+
});
|
|
19
|
+
console.log(`synced ${result.uploadedCount} R2 object(s), skipped ${result.skippedCount}, state written to ${result.stateFile}`);
|
|
20
|
+
}
|
|
21
|
+
function parseArgs(argv) {
|
|
22
|
+
const result = {};
|
|
23
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
24
|
+
const argument = argv[index];
|
|
25
|
+
const nextValue = argv[index + 1];
|
|
26
|
+
if (argument === '--help' || argument === '-h') {
|
|
27
|
+
result.help = true;
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
if (argument === '--dir' && nextValue) {
|
|
31
|
+
result.dir = nextValue;
|
|
32
|
+
index += 1;
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
if (argument === '--bucket' && nextValue) {
|
|
36
|
+
result.bucket = nextValue;
|
|
37
|
+
index += 1;
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
if (argument === '--force') {
|
|
41
|
+
result.force = true;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
throw new Error(`Unknown argument for mdorigin sync cloudflare-r2: ${argument}`);
|
|
45
|
+
}
|
|
46
|
+
return result;
|
|
47
|
+
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export { createCloudflareWorker } from './adapters/cloudflare.js';
|
|
2
|
-
export type { CloudflareManifest, CloudflareManifestEntry, ExportedHandlerLike, } from './adapters/cloudflare.js';
|
|
2
|
+
export type { CloudflareBundleRuntimeConfig, CloudflareManifest, CloudflareManifestEntry, ExportedHandlerLike, } from './adapters/cloudflare.js';
|
package/dist/cloudflare.d.ts
CHANGED
|
@@ -1,34 +1,72 @@
|
|
|
1
|
-
import type { CloudflareManifest, CloudflareManifestEntry } from './adapters/cloudflare.js';
|
|
1
|
+
import type { CloudflareBundleRuntimeConfig, CloudflareManifest, CloudflareManifestEntry } from './adapters/cloudflare.js';
|
|
2
2
|
import { createCloudflareWorker } from './adapters/cloudflare.js';
|
|
3
3
|
import type { ResolvedSiteConfig } from './core/site-config.js';
|
|
4
4
|
export { createCloudflareWorker };
|
|
5
|
-
export type { CloudflareManifest, CloudflareManifestEntry };
|
|
5
|
+
export type { CloudflareBundleRuntimeConfig, CloudflareManifest, CloudflareManifestEntry, };
|
|
6
|
+
export type CloudflareBinaryMode = 'inline' | 'external';
|
|
6
7
|
export interface BuildCloudflareManifestOptions {
|
|
7
8
|
rootDir: string;
|
|
8
9
|
siteConfig: ResolvedSiteConfig;
|
|
9
10
|
searchDir?: string;
|
|
11
|
+
binaryMode?: CloudflareBinaryMode;
|
|
12
|
+
assetsMaxBytes?: number;
|
|
13
|
+
r2Binding?: string;
|
|
10
14
|
}
|
|
11
|
-
export interface WriteCloudflareBundleOptions {
|
|
12
|
-
rootDir: string;
|
|
15
|
+
export interface WriteCloudflareBundleOptions extends BuildCloudflareManifestOptions {
|
|
13
16
|
outDir: string;
|
|
14
|
-
siteConfig: ResolvedSiteConfig;
|
|
15
17
|
packageImport?: string;
|
|
16
|
-
searchDir?: string;
|
|
17
18
|
configModulePath?: string;
|
|
18
19
|
}
|
|
20
|
+
export interface CloudflareBundleMetadata {
|
|
21
|
+
version: 2;
|
|
22
|
+
workerEntry: string;
|
|
23
|
+
binaryMode: CloudflareBinaryMode;
|
|
24
|
+
assetsMaxBytes?: number;
|
|
25
|
+
assetsDir?: string;
|
|
26
|
+
r2Dir?: string;
|
|
27
|
+
r2Binding?: string;
|
|
28
|
+
siteTitle?: string;
|
|
29
|
+
stagedObjects: Array<{
|
|
30
|
+
kind: 'binary' | 'search';
|
|
31
|
+
path: string;
|
|
32
|
+
mediaType: string;
|
|
33
|
+
storageKind: 'assets' | 'r2';
|
|
34
|
+
storageKey: string;
|
|
35
|
+
file: string;
|
|
36
|
+
byteSize: number;
|
|
37
|
+
}>;
|
|
38
|
+
}
|
|
19
39
|
export interface InitCloudflareProjectOptions {
|
|
20
40
|
projectDir: string;
|
|
21
41
|
workerEntry: string;
|
|
22
42
|
workerName?: string;
|
|
23
43
|
siteTitle?: string;
|
|
24
44
|
compatibilityDate?: string;
|
|
45
|
+
r2Bucket?: string;
|
|
46
|
+
force?: boolean;
|
|
47
|
+
}
|
|
48
|
+
export interface SyncCloudflareR2Options {
|
|
49
|
+
dir: string;
|
|
50
|
+
bucketName: string;
|
|
25
51
|
force?: boolean;
|
|
52
|
+
runCommand?: typeof runWranglerCommand;
|
|
53
|
+
}
|
|
54
|
+
export interface SyncCloudflareR2Result {
|
|
55
|
+
uploadedCount: number;
|
|
56
|
+
skippedCount: number;
|
|
57
|
+
stateFile: string;
|
|
26
58
|
}
|
|
27
59
|
export declare function buildCloudflareManifest(options: BuildCloudflareManifestOptions): Promise<CloudflareManifest>;
|
|
28
60
|
export declare function writeCloudflareBundle(options: WriteCloudflareBundleOptions): Promise<{
|
|
29
61
|
manifest: CloudflareManifest;
|
|
30
62
|
workerFile: string;
|
|
63
|
+
bundleFile: string;
|
|
31
64
|
}>;
|
|
32
65
|
export declare function initCloudflareProject(options: InitCloudflareProjectOptions): Promise<{
|
|
33
66
|
configFile: string;
|
|
34
67
|
}>;
|
|
68
|
+
export declare function syncCloudflareR2(options: SyncCloudflareR2Options): Promise<SyncCloudflareR2Result>;
|
|
69
|
+
declare function runWranglerCommand(command: string, args: string[]): {
|
|
70
|
+
status: number | null;
|
|
71
|
+
stderr: string;
|
|
72
|
+
};
|
package/dist/cloudflare.js
CHANGED
|
@@ -1,12 +1,24 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { spawnSync } from 'node:child_process';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { createReadStream } from 'node:fs';
|
|
4
|
+
import { copyFile, mkdir, readFile, readdir, realpath, rm, stat, writeFile, } from 'node:fs/promises';
|
|
2
5
|
import path from 'node:path';
|
|
3
6
|
import { createCloudflareWorker } from './adapters/cloudflare.js';
|
|
4
|
-
import { getMediaTypeForPath, isLikelyTextPath, normalizeContentPath, } from './core/content-store.js';
|
|
7
|
+
import { getMediaTypeForPath, isIgnoredContentName, isLikelyTextPath, normalizeContentPath, } from './core/content-store.js';
|
|
5
8
|
export { createCloudflareWorker };
|
|
9
|
+
const DEFAULT_ASSETS_MAX_BYTES = 25 * 1024 * 1024;
|
|
10
|
+
const DEFAULT_ASSETS_BINDING = 'ASSETS';
|
|
11
|
+
const DEFAULT_R2_BINDING = 'MDORIGIN_R2';
|
|
12
|
+
const BUNDLE_FILE_NAME = 'bundle.json';
|
|
13
|
+
const R2_STATE_FILE_NAME = 'r2-sync-state.json';
|
|
14
|
+
const SEARCH_ASSETS_PREFIX = '__mdorigin/search';
|
|
6
15
|
export async function buildCloudflareManifest(options) {
|
|
7
16
|
const rootDir = path.resolve(options.rootDir);
|
|
8
17
|
const files = await listFiles(rootDir);
|
|
9
18
|
const entries = [];
|
|
19
|
+
const binaryMode = options.binaryMode ?? 'inline';
|
|
20
|
+
const assetsMaxBytes = options.assetsMaxBytes ?? DEFAULT_ASSETS_MAX_BYTES;
|
|
21
|
+
const r2Binding = options.r2Binding ?? DEFAULT_R2_BINDING;
|
|
10
22
|
for (const filePath of files) {
|
|
11
23
|
const relativePath = path.relative(rootDir, filePath).replaceAll(path.sep, '/');
|
|
12
24
|
const normalizedPath = normalizeContentPath(relativePath);
|
|
@@ -23,32 +35,56 @@ export async function buildCloudflareManifest(options) {
|
|
|
23
35
|
});
|
|
24
36
|
continue;
|
|
25
37
|
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
38
|
+
if (binaryMode === 'inline') {
|
|
39
|
+
const bytes = await readFile(filePath);
|
|
40
|
+
entries.push({
|
|
41
|
+
path: normalizedPath,
|
|
42
|
+
kind: 'binary',
|
|
43
|
+
mediaType,
|
|
44
|
+
base64: bytes.toString('base64'),
|
|
45
|
+
});
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const fileStats = await stat(filePath);
|
|
49
|
+
entries.push(await buildExternalBinaryEntry(filePath, normalizedPath, mediaType, fileStats.size, {
|
|
50
|
+
assetsMaxBytes,
|
|
51
|
+
r2Binding,
|
|
52
|
+
}));
|
|
32
53
|
}
|
|
33
|
-
const
|
|
34
|
-
? await
|
|
54
|
+
const externalSearchEntries = options.searchDir
|
|
55
|
+
? await buildSearchBundleEntries(path.resolve(options.searchDir), assetsMaxBytes)
|
|
35
56
|
: undefined;
|
|
36
57
|
entries.sort((left, right) => left.path.localeCompare(right.path));
|
|
37
58
|
return {
|
|
38
59
|
entries,
|
|
39
60
|
siteConfig: options.siteConfig,
|
|
40
|
-
|
|
61
|
+
externalSearchEntries,
|
|
62
|
+
runtime: binaryMode === 'external' || (externalSearchEntries?.length ?? 0) > 0
|
|
63
|
+
? {
|
|
64
|
+
binaryMode,
|
|
65
|
+
r2Binding,
|
|
66
|
+
}
|
|
67
|
+
: {
|
|
68
|
+
binaryMode,
|
|
69
|
+
},
|
|
41
70
|
};
|
|
42
71
|
}
|
|
43
72
|
export async function writeCloudflareBundle(options) {
|
|
44
73
|
const outDir = path.resolve(options.outDir);
|
|
74
|
+
const binaryMode = options.binaryMode ?? 'inline';
|
|
75
|
+
const assetsMaxBytes = options.assetsMaxBytes ?? DEFAULT_ASSETS_MAX_BYTES;
|
|
76
|
+
const r2Binding = options.r2Binding ?? DEFAULT_R2_BINDING;
|
|
45
77
|
const manifest = await buildCloudflareManifest({
|
|
46
78
|
rootDir: options.rootDir,
|
|
47
79
|
siteConfig: options.siteConfig,
|
|
48
80
|
searchDir: options.searchDir,
|
|
81
|
+
binaryMode,
|
|
82
|
+
assetsMaxBytes,
|
|
83
|
+
r2Binding,
|
|
49
84
|
});
|
|
50
85
|
const packageImport = options.packageImport ?? 'mdorigin/cloudflare-runtime';
|
|
51
86
|
const workerFile = path.join(outDir, 'worker.mjs');
|
|
87
|
+
const bundleFile = path.join(outDir, BUNDLE_FILE_NAME);
|
|
52
88
|
const configImportPath = options.configModulePath
|
|
53
89
|
? toPosixPath(path.relative(outDir, options.configModulePath))
|
|
54
90
|
: null;
|
|
@@ -83,10 +119,18 @@ export async function writeCloudflareBundle(options) {
|
|
|
83
119
|
'',
|
|
84
120
|
].join('\n');
|
|
85
121
|
await mkdir(outDir, { recursive: true });
|
|
122
|
+
const metadata = await writeExternalStaging(path.resolve(options.rootDir), options.searchDir ? path.resolve(options.searchDir) : undefined, outDir, manifest, {
|
|
123
|
+
binaryMode,
|
|
124
|
+
assetsMaxBytes,
|
|
125
|
+
r2Binding,
|
|
126
|
+
siteTitle: options.siteConfig.siteTitle,
|
|
127
|
+
});
|
|
86
128
|
await writeFile(workerFile, workerSource, 'utf8');
|
|
129
|
+
await writeFile(bundleFile, JSON.stringify(metadata, null, 2), 'utf8');
|
|
87
130
|
return {
|
|
88
131
|
manifest,
|
|
89
132
|
workerFile,
|
|
133
|
+
bundleFile,
|
|
90
134
|
};
|
|
91
135
|
}
|
|
92
136
|
export async function initCloudflareProject(options) {
|
|
@@ -96,8 +140,14 @@ export async function initCloudflareProject(options) {
|
|
|
96
140
|
if (existing && !options.force) {
|
|
97
141
|
throw new Error(`Refusing to overwrite ${configFile}. Re-run with --force to replace it.`);
|
|
98
142
|
}
|
|
143
|
+
const bundleMetadata = await readCloudflareBundleMetadata(options.workerEntry);
|
|
144
|
+
if (bundleMetadata &&
|
|
145
|
+
bundleMetadata.stagedObjects.some((object) => object.storageKind === 'r2') &&
|
|
146
|
+
!options.r2Bucket) {
|
|
147
|
+
throw new Error('Cloudflare bundle contains R2-backed staged objects. Re-run init cloudflare with --r2-bucket <bucket-name>.');
|
|
148
|
+
}
|
|
99
149
|
const workerName = options.workerName ??
|
|
100
|
-
slugifyWorkerName(options.siteTitle) ??
|
|
150
|
+
slugifyWorkerName(bundleMetadata?.siteTitle ?? options.siteTitle) ??
|
|
101
151
|
'mdorigin-site';
|
|
102
152
|
const compatibilityDate = options.compatibilityDate ?? '2026-03-20';
|
|
103
153
|
const wranglerConfig = [
|
|
@@ -107,13 +157,273 @@ export async function initCloudflareProject(options) {
|
|
|
107
157
|
` "main": ${JSON.stringify(toPosixPath(path.relative(projectDir, options.workerEntry)))},`,
|
|
108
158
|
` "compatibility_date": ${JSON.stringify(compatibilityDate)},`,
|
|
109
159
|
' "compatibility_flags": ["nodejs_compat"]',
|
|
160
|
+
bundleMetadata?.assetsDir
|
|
161
|
+
? [
|
|
162
|
+
',',
|
|
163
|
+
' "assets": {',
|
|
164
|
+
` "directory": ${JSON.stringify(toPosixPath(path.relative(projectDir, path.join(path.dirname(options.workerEntry), bundleMetadata.assetsDir))))},`,
|
|
165
|
+
` "binding": ${JSON.stringify(DEFAULT_ASSETS_BINDING)},`,
|
|
166
|
+
' "run_worker_first": true',
|
|
167
|
+
' }',
|
|
168
|
+
].join('\n')
|
|
169
|
+
: '',
|
|
170
|
+
bundleMetadata &&
|
|
171
|
+
bundleMetadata.stagedObjects.some((object) => object.storageKind === 'r2') &&
|
|
172
|
+
bundleMetadata.r2Binding &&
|
|
173
|
+
options.r2Bucket
|
|
174
|
+
? [
|
|
175
|
+
',',
|
|
176
|
+
' "r2_buckets": [',
|
|
177
|
+
' {',
|
|
178
|
+
` "binding": ${JSON.stringify(bundleMetadata.r2Binding)},`,
|
|
179
|
+
` "bucket_name": ${JSON.stringify(options.r2Bucket)}`,
|
|
180
|
+
' }',
|
|
181
|
+
' ]',
|
|
182
|
+
].join('\n')
|
|
183
|
+
: '',
|
|
110
184
|
'}',
|
|
111
185
|
'',
|
|
112
|
-
]
|
|
186
|
+
]
|
|
187
|
+
.filter(Boolean)
|
|
188
|
+
.join('\n');
|
|
113
189
|
await mkdir(projectDir, { recursive: true });
|
|
114
190
|
await writeFile(configFile, wranglerConfig, 'utf8');
|
|
115
191
|
return { configFile };
|
|
116
192
|
}
|
|
193
|
+
export async function syncCloudflareR2(options) {
|
|
194
|
+
const outDir = path.resolve(options.dir);
|
|
195
|
+
const bundleFile = path.join(outDir, BUNDLE_FILE_NAME);
|
|
196
|
+
const metadata = await readBundleMetadataFile(bundleFile);
|
|
197
|
+
const r2Objects = metadata.stagedObjects.filter((object) => object.storageKind === 'r2');
|
|
198
|
+
if (r2Objects.length === 0) {
|
|
199
|
+
throw new Error(`No R2-backed staged objects found in ${bundleFile}.`);
|
|
200
|
+
}
|
|
201
|
+
const stateFile = path.join(outDir, R2_STATE_FILE_NAME);
|
|
202
|
+
const state = await readR2SyncState(stateFile);
|
|
203
|
+
const runCommand = options.runCommand ?? runWranglerCommand;
|
|
204
|
+
let uploadedCount = 0;
|
|
205
|
+
let skippedCount = 0;
|
|
206
|
+
for (const object of r2Objects) {
|
|
207
|
+
const stateKey = `${options.bucketName}:${object.storageKey}`;
|
|
208
|
+
if (!options.force && state.uploaded[stateKey]) {
|
|
209
|
+
skippedCount += 1;
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
const filePath = path.join(outDir, object.file);
|
|
213
|
+
const result = runCommand('wrangler', [
|
|
214
|
+
'r2',
|
|
215
|
+
'object',
|
|
216
|
+
'put',
|
|
217
|
+
`${options.bucketName}/${object.storageKey}`,
|
|
218
|
+
'--file',
|
|
219
|
+
filePath,
|
|
220
|
+
'--content-type',
|
|
221
|
+
object.mediaType,
|
|
222
|
+
'--remote',
|
|
223
|
+
]);
|
|
224
|
+
if (result.status !== 0) {
|
|
225
|
+
throw new Error(result.stderr || `Failed to upload R2 object ${object.storageKey}.`);
|
|
226
|
+
}
|
|
227
|
+
state.uploaded[stateKey] = {
|
|
228
|
+
syncedAt: new Date().toISOString(),
|
|
229
|
+
};
|
|
230
|
+
uploadedCount += 1;
|
|
231
|
+
}
|
|
232
|
+
await writeFile(stateFile, JSON.stringify(state, null, 2), 'utf8');
|
|
233
|
+
return {
|
|
234
|
+
uploadedCount,
|
|
235
|
+
skippedCount,
|
|
236
|
+
stateFile,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
async function buildExternalBinaryEntry(filePath, normalizedPath, mediaType, byteSize, options) {
|
|
240
|
+
if (byteSize <= options.assetsMaxBytes) {
|
|
241
|
+
return {
|
|
242
|
+
path: normalizedPath,
|
|
243
|
+
kind: 'binary',
|
|
244
|
+
mediaType,
|
|
245
|
+
storageKind: 'assets',
|
|
246
|
+
storageKey: normalizedPath,
|
|
247
|
+
byteSize,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
return {
|
|
251
|
+
path: normalizedPath,
|
|
252
|
+
kind: 'binary',
|
|
253
|
+
mediaType,
|
|
254
|
+
storageKind: 'r2',
|
|
255
|
+
storageKey: await buildR2StorageKey(filePath, normalizedPath),
|
|
256
|
+
byteSize,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
async function writeExternalStaging(rootDir, searchDir, outDir, manifest, options) {
|
|
260
|
+
const assetsDir = path.join(outDir, 'assets');
|
|
261
|
+
const r2Dir = path.join(outDir, 'r2');
|
|
262
|
+
await rm(assetsDir, { recursive: true, force: true });
|
|
263
|
+
await rm(r2Dir, { recursive: true, force: true });
|
|
264
|
+
const stagedObjects = new Map();
|
|
265
|
+
let hasAssets = false;
|
|
266
|
+
let hasR2 = false;
|
|
267
|
+
if (options.binaryMode === 'external') {
|
|
268
|
+
for (const entry of manifest.entries) {
|
|
269
|
+
if (entry.kind !== 'binary' || !('storageKind' in entry)) {
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
const sourceFile = path.resolve(rootDir, entry.path);
|
|
273
|
+
if (entry.storageKind === 'assets') {
|
|
274
|
+
const targetFile = path.join(assetsDir, entry.storageKey);
|
|
275
|
+
await mkdir(path.dirname(targetFile), { recursive: true });
|
|
276
|
+
await copyFile(sourceFile, targetFile);
|
|
277
|
+
hasAssets = true;
|
|
278
|
+
stagedObjects.set(`assets:${entry.storageKey}`, {
|
|
279
|
+
kind: 'binary',
|
|
280
|
+
path: entry.path,
|
|
281
|
+
mediaType: entry.mediaType,
|
|
282
|
+
storageKind: 'assets',
|
|
283
|
+
storageKey: entry.storageKey,
|
|
284
|
+
file: toPosixPath(path.join('assets', entry.storageKey)),
|
|
285
|
+
byteSize: entry.byteSize,
|
|
286
|
+
});
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
289
|
+
const relativeFile = toPosixPath(path.join('r2', entry.storageKey));
|
|
290
|
+
const targetFile = path.join(outDir, relativeFile);
|
|
291
|
+
if (!stagedObjects.has(`r2:${entry.storageKey}`)) {
|
|
292
|
+
await mkdir(path.dirname(targetFile), { recursive: true });
|
|
293
|
+
await copyFile(sourceFile, targetFile);
|
|
294
|
+
hasR2 = true;
|
|
295
|
+
stagedObjects.set(`r2:${entry.storageKey}`, {
|
|
296
|
+
kind: 'binary',
|
|
297
|
+
path: entry.path,
|
|
298
|
+
mediaType: entry.mediaType,
|
|
299
|
+
storageKind: 'r2',
|
|
300
|
+
storageKey: entry.storageKey,
|
|
301
|
+
file: relativeFile,
|
|
302
|
+
byteSize: entry.byteSize,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (searchDir && manifest.externalSearchEntries) {
|
|
308
|
+
for (const entry of manifest.externalSearchEntries) {
|
|
309
|
+
const sourceFile = path.join(searchDir, entry.path);
|
|
310
|
+
if (entry.storageKind === 'assets') {
|
|
311
|
+
const targetFile = path.join(assetsDir, entry.storageKey);
|
|
312
|
+
await mkdir(path.dirname(targetFile), { recursive: true });
|
|
313
|
+
await copyFile(sourceFile, targetFile);
|
|
314
|
+
hasAssets = true;
|
|
315
|
+
stagedObjects.set(`assets:${entry.storageKey}`, {
|
|
316
|
+
kind: 'search',
|
|
317
|
+
path: entry.path,
|
|
318
|
+
mediaType: entry.mediaType,
|
|
319
|
+
storageKind: 'assets',
|
|
320
|
+
storageKey: entry.storageKey,
|
|
321
|
+
file: toPosixPath(path.join('assets', entry.storageKey)),
|
|
322
|
+
byteSize: entry.byteSize,
|
|
323
|
+
});
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
const relativeFile = toPosixPath(path.join('r2', entry.storageKey));
|
|
327
|
+
const targetFile = path.join(outDir, relativeFile);
|
|
328
|
+
if (!stagedObjects.has(`r2:${entry.storageKey}`)) {
|
|
329
|
+
await mkdir(path.dirname(targetFile), { recursive: true });
|
|
330
|
+
await copyFile(sourceFile, targetFile);
|
|
331
|
+
hasR2 = true;
|
|
332
|
+
stagedObjects.set(`r2:${entry.storageKey}`, {
|
|
333
|
+
kind: 'search',
|
|
334
|
+
path: entry.path,
|
|
335
|
+
mediaType: entry.mediaType,
|
|
336
|
+
storageKind: 'r2',
|
|
337
|
+
storageKey: entry.storageKey,
|
|
338
|
+
file: relativeFile,
|
|
339
|
+
byteSize: entry.byteSize,
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
return {
|
|
345
|
+
version: 2,
|
|
346
|
+
workerEntry: 'worker.mjs',
|
|
347
|
+
binaryMode: options.binaryMode,
|
|
348
|
+
assetsMaxBytes: options.binaryMode === 'external' || searchDir ? options.assetsMaxBytes : undefined,
|
|
349
|
+
assetsDir: hasAssets ? 'assets' : undefined,
|
|
350
|
+
r2Dir: hasR2 ? 'r2' : undefined,
|
|
351
|
+
r2Binding: hasR2 ? options.r2Binding : undefined,
|
|
352
|
+
siteTitle: options.siteTitle,
|
|
353
|
+
stagedObjects: Array.from(stagedObjects.values()).sort((left, right) => left.storageKey.localeCompare(right.storageKey)),
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
async function readCloudflareBundleMetadata(workerEntry) {
|
|
357
|
+
const bundleFile = path.join(path.dirname(workerEntry), BUNDLE_FILE_NAME);
|
|
358
|
+
if (!(await pathExists(bundleFile))) {
|
|
359
|
+
return null;
|
|
360
|
+
}
|
|
361
|
+
return readBundleMetadataFile(bundleFile);
|
|
362
|
+
}
|
|
363
|
+
async function readBundleMetadataFile(bundleFile) {
|
|
364
|
+
const parsed = JSON.parse(await readFile(bundleFile, 'utf8'));
|
|
365
|
+
if ('stagedObjects' in parsed && Array.isArray(parsed.stagedObjects)) {
|
|
366
|
+
return parsed;
|
|
367
|
+
}
|
|
368
|
+
if ('r2Objects' in parsed && Array.isArray(parsed.r2Objects)) {
|
|
369
|
+
return {
|
|
370
|
+
version: 2,
|
|
371
|
+
workerEntry: parsed.workerEntry,
|
|
372
|
+
binaryMode: parsed.binaryMode,
|
|
373
|
+
assetsMaxBytes: parsed.assetsMaxBytes,
|
|
374
|
+
assetsDir: parsed.assetsDir,
|
|
375
|
+
r2Dir: parsed.r2Dir,
|
|
376
|
+
r2Binding: parsed.r2Binding,
|
|
377
|
+
siteTitle: parsed.siteTitle,
|
|
378
|
+
stagedObjects: parsed.r2Objects.map((object) => ({
|
|
379
|
+
kind: 'binary',
|
|
380
|
+
path: object.path,
|
|
381
|
+
mediaType: object.mediaType,
|
|
382
|
+
storageKind: 'r2',
|
|
383
|
+
storageKey: object.storageKey,
|
|
384
|
+
file: object.file,
|
|
385
|
+
byteSize: object.byteSize,
|
|
386
|
+
})),
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
throw new Error(`Bundle metadata in ${bundleFile} is not supported. Rebuild the Cloudflare bundle with the current mdorigin CLI.`);
|
|
390
|
+
}
|
|
391
|
+
async function readR2SyncState(stateFile) {
|
|
392
|
+
if (!(await pathExists(stateFile))) {
|
|
393
|
+
return { version: 1, uploaded: {} };
|
|
394
|
+
}
|
|
395
|
+
return JSON.parse(await readFile(stateFile, 'utf8'));
|
|
396
|
+
}
|
|
397
|
+
async function buildR2StorageKey(filePath, normalizedPath) {
|
|
398
|
+
const extension = path.posix.extname(normalizedPath).toLowerCase();
|
|
399
|
+
const hash = await hashFile(filePath);
|
|
400
|
+
return extension ? `binary/${hash}${extension}` : `binary/${hash}`;
|
|
401
|
+
}
|
|
402
|
+
async function hashFile(filePath) {
|
|
403
|
+
const hash = createHash('sha256');
|
|
404
|
+
await new Promise((resolve, reject) => {
|
|
405
|
+
const stream = createReadStream(filePath);
|
|
406
|
+
stream.on('data', (chunk) => {
|
|
407
|
+
hash.update(chunk);
|
|
408
|
+
});
|
|
409
|
+
stream.on('end', () => resolve());
|
|
410
|
+
stream.on('error', reject);
|
|
411
|
+
});
|
|
412
|
+
return hash.digest('hex');
|
|
413
|
+
}
|
|
414
|
+
function runWranglerCommand(command, args) {
|
|
415
|
+
const result = spawnSync(command, args, {
|
|
416
|
+
encoding: 'utf8',
|
|
417
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
418
|
+
});
|
|
419
|
+
if (result.error) {
|
|
420
|
+
throw result.error;
|
|
421
|
+
}
|
|
422
|
+
return {
|
|
423
|
+
status: result.status,
|
|
424
|
+
stderr: result.stderr ?? '',
|
|
425
|
+
};
|
|
426
|
+
}
|
|
117
427
|
async function listFiles(directory, visitedRealDirectories = new Set()) {
|
|
118
428
|
const directoryRealPath = await realpath(directory);
|
|
119
429
|
if (visitedRealDirectories.has(directoryRealPath)) {
|
|
@@ -123,6 +433,9 @@ async function listFiles(directory, visitedRealDirectories = new Set()) {
|
|
|
123
433
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
124
434
|
const files = [];
|
|
125
435
|
for (const entry of entries) {
|
|
436
|
+
if (isIgnoredContentName(entry.name)) {
|
|
437
|
+
continue;
|
|
438
|
+
}
|
|
126
439
|
const fullPath = path.join(directory, entry.name);
|
|
127
440
|
const entryStats = await stat(fullPath);
|
|
128
441
|
if (entryStats.isDirectory()) {
|
|
@@ -135,31 +448,39 @@ async function listFiles(directory, visitedRealDirectories = new Set()) {
|
|
|
135
448
|
}
|
|
136
449
|
return files;
|
|
137
450
|
}
|
|
138
|
-
async function
|
|
451
|
+
async function buildSearchBundleEntries(directory, assetsMaxBytes) {
|
|
139
452
|
const files = await listFiles(directory);
|
|
140
453
|
const entries = [];
|
|
141
454
|
for (const filePath of files) {
|
|
142
455
|
const relativePath = path.relative(directory, filePath).replaceAll(path.sep, '/');
|
|
143
456
|
const mediaType = getMediaTypeForPath(relativePath);
|
|
144
|
-
|
|
457
|
+
const fileStats = await stat(filePath);
|
|
458
|
+
if (fileStats.size <= assetsMaxBytes) {
|
|
145
459
|
entries.push({
|
|
146
460
|
path: relativePath,
|
|
147
|
-
kind: 'text',
|
|
148
461
|
mediaType,
|
|
149
|
-
|
|
462
|
+
storageKind: 'assets',
|
|
463
|
+
storageKey: `${SEARCH_ASSETS_PREFIX}/${relativePath}`,
|
|
464
|
+
byteSize: fileStats.size,
|
|
150
465
|
});
|
|
151
466
|
continue;
|
|
152
467
|
}
|
|
153
468
|
entries.push({
|
|
154
469
|
path: relativePath,
|
|
155
|
-
kind: 'binary',
|
|
156
470
|
mediaType,
|
|
157
|
-
|
|
471
|
+
storageKind: 'r2',
|
|
472
|
+
storageKey: await buildSearchStorageKey(filePath, relativePath),
|
|
473
|
+
byteSize: fileStats.size,
|
|
158
474
|
});
|
|
159
475
|
}
|
|
160
476
|
entries.sort((left, right) => left.path.localeCompare(right.path));
|
|
161
477
|
return entries;
|
|
162
478
|
}
|
|
479
|
+
async function buildSearchStorageKey(filePath, relativePath) {
|
|
480
|
+
const extension = path.posix.extname(relativePath).toLowerCase();
|
|
481
|
+
const hash = await hashFile(filePath);
|
|
482
|
+
return extension ? `search/${hash}${extension}` : `search/${hash}`;
|
|
483
|
+
}
|
|
163
484
|
async function pathExists(filePath) {
|
|
164
485
|
try {
|
|
165
486
|
await stat(filePath);
|
package/dist/core/api.js
CHANGED
|
@@ -13,10 +13,15 @@ export async function handleApiRoute(pathname, searchParams, options) {
|
|
|
13
13
|
});
|
|
14
14
|
}
|
|
15
15
|
const topK = normalizePositiveInteger(searchParams?.get('topK')) ?? 10;
|
|
16
|
-
const
|
|
16
|
+
const metadata = readSearchMetadataFilters(searchParams);
|
|
17
|
+
const hits = await options.searchApi.search(query, {
|
|
18
|
+
topK,
|
|
19
|
+
metadata,
|
|
20
|
+
});
|
|
17
21
|
return json(200, {
|
|
18
22
|
query,
|
|
19
23
|
topK,
|
|
24
|
+
metadata,
|
|
20
25
|
count: hits.length,
|
|
21
26
|
hits: hits.map(serializeSearchHit),
|
|
22
27
|
});
|
|
@@ -56,6 +61,13 @@ function buildOpenApiDocument(options) {
|
|
|
56
61
|
schema: { type: 'integer', minimum: 1, default: 10 },
|
|
57
62
|
description: 'Maximum number of hits to return.',
|
|
58
63
|
},
|
|
64
|
+
{
|
|
65
|
+
name: 'meta.<field>',
|
|
66
|
+
in: 'query',
|
|
67
|
+
required: false,
|
|
68
|
+
schema: { type: 'string' },
|
|
69
|
+
description: 'Exact-match metadata filter. Use query parameters such as meta.type=post or meta.section=guides.',
|
|
70
|
+
},
|
|
59
71
|
],
|
|
60
72
|
responses: {
|
|
61
73
|
'200': {
|
|
@@ -68,6 +80,10 @@ function buildOpenApiDocument(options) {
|
|
|
68
80
|
properties: {
|
|
69
81
|
query: { type: 'string' },
|
|
70
82
|
topK: { type: 'integer' },
|
|
83
|
+
metadata: {
|
|
84
|
+
type: 'object',
|
|
85
|
+
additionalProperties: { type: 'string' },
|
|
86
|
+
},
|
|
71
87
|
count: { type: 'integer' },
|
|
72
88
|
hits: {
|
|
73
89
|
type: 'array',
|
|
@@ -130,6 +146,23 @@ function buildOpenApiDocument(options) {
|
|
|
130
146
|
},
|
|
131
147
|
};
|
|
132
148
|
}
|
|
149
|
+
function readSearchMetadataFilters(searchParams) {
|
|
150
|
+
if (!searchParams) {
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
const metadata = {};
|
|
154
|
+
for (const [key, value] of searchParams.entries()) {
|
|
155
|
+
if (!key.startsWith('meta.') || value.trim() === '') {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
const metadataKey = key.slice('meta.'.length).trim();
|
|
159
|
+
if (metadataKey === '') {
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
metadata[metadataKey] = value;
|
|
163
|
+
}
|
|
164
|
+
return Object.keys(metadata).length > 0 ? metadata : undefined;
|
|
165
|
+
}
|
|
133
166
|
function serializeSearchHit(hit) {
|
|
134
167
|
return {
|
|
135
168
|
docId: hit.docId,
|
|
@@ -15,6 +15,7 @@ export interface ContentStore {
|
|
|
15
15
|
get(contentPath: string): Promise<ContentEntry | null>;
|
|
16
16
|
listDirectory(contentPath: string): Promise<ContentDirectoryEntry[] | null>;
|
|
17
17
|
}
|
|
18
|
+
export declare function isIgnoredContentName(name: string): boolean;
|
|
18
19
|
export declare class MemoryContentStore implements ContentStore {
|
|
19
20
|
private readonly entries;
|
|
20
21
|
constructor(entries: Iterable<ContentEntry>);
|