mdorigin 0.1.8 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/adapters/cloudflare.d.ts +42 -5
- package/dist/adapters/cloudflare.js +250 -10
- package/dist/adapters/node.js +2 -2
- package/dist/cli/build-cloudflare.js +27 -2
- package/dist/cli/build-search.js +15 -4
- package/dist/cli/help.js +8 -7
- package/dist/cli/init-cloudflare.js +7 -1
- package/dist/cli/main.js +5 -0
- package/dist/cli/search.js +16 -4
- package/dist/cli/sync-cloudflare-r2.d.ts +1 -0
- package/dist/cli/sync-cloudflare-r2.js +47 -0
- package/dist/cloudflare-runtime.d.ts +1 -1
- package/dist/cloudflare.d.ts +44 -6
- package/dist/cloudflare.js +340 -19
- package/dist/core/api.js +34 -1
- package/dist/core/content-store.d.ts +1 -0
- package/dist/core/content-store.js +8 -1
- package/dist/core/extensions.d.ts +5 -9
- package/dist/core/markdown.d.ts +1 -0
- package/dist/core/markdown.js +9 -2
- package/dist/core/request-handler.js +46 -60
- package/dist/core/site-config.d.ts +4 -10
- package/dist/core/site-config.js +18 -10
- package/dist/html/template.d.ts +5 -9
- package/dist/html/template.js +25 -25
- package/dist/html/theme.d.ts +1 -2
- package/dist/html/theme.js +2 -642
- package/dist/index-builder.js +5 -4
- package/dist/search.d.ts +27 -4
- package/dist/search.js +301 -38
- package/package.json +5 -3
- package/dist/html/template-kind.d.ts +0 -1
- package/dist/html/template-kind.js +0 -1
package/README.md
CHANGED
|
@@ -108,6 +108,18 @@ mdorigin dev --root docs/site --search dist/search
|
|
|
108
108
|
mdorigin build cloudflare --root docs/site --search dist/search
|
|
109
109
|
```
|
|
110
110
|
|
|
111
|
+
For media-heavy sites that would exceed Worker bundle limits, build in external binary mode instead:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
mdorigin build cloudflare --root docs/site --binary-mode external
|
|
115
|
+
mdorigin sync cloudflare-r2 --dir dist/cloudflare --bucket <bucket-name>
|
|
116
|
+
mdorigin init cloudflare --dir . --r2-bucket <bucket-name>
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
In this mode, markdown and other text stay embedded in the Worker bundle, smaller binaries are staged for Workers Static Assets, and oversized binaries are staged for R2.
|
|
120
|
+
|
|
121
|
+
`mdorigin` ignores dotfiles and dot-directories during content traversal. `.gitignore` is not used to decide publish visibility.
|
|
122
|
+
|
|
111
123
|
Runtime endpoints:
|
|
112
124
|
|
|
113
125
|
- `/api/search?q=cloudflare+deploy`
|
|
@@ -123,3 +135,5 @@ Runtime endpoints:
|
|
|
123
135
|
- CLI: [`docs/site/reference/cli.md`](docs/site/reference/cli.md)
|
|
124
136
|
- Search setup: [`docs/site/guides/getting-started.md`](docs/site/guides/getting-started.md#quick-start)
|
|
125
137
|
- Cloudflare deployment: [`docs/site/guides/cloudflare.md`](docs/site/guides/cloudflare.md)
|
|
138
|
+
|
|
139
|
+
The docs site at <https://mdorigin.jolestar.workers.dev> is deployed automatically from `main` with GitHub Actions.
|
|
@@ -1,22 +1,59 @@
|
|
|
1
|
-
import { type ContentEntryKind } from '../core/content-store.js';
|
|
2
1
|
import type { MdoPlugin } from '../core/extensions.js';
|
|
3
2
|
import type { ResolvedSiteConfig } from '../core/site-config.js';
|
|
4
|
-
import { type SearchBundleEntry } from '../search.js';
|
|
5
|
-
export interface
|
|
3
|
+
import { type ExternalSearchBundleEntry, type SearchBundleEntry } from '../search.js';
|
|
4
|
+
export interface TextCloudflareManifestEntry {
|
|
6
5
|
path: string;
|
|
7
|
-
kind:
|
|
6
|
+
kind: 'text';
|
|
8
7
|
mediaType: string;
|
|
9
8
|
text?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface InlineBinaryCloudflareManifestEntry {
|
|
11
|
+
path: string;
|
|
12
|
+
kind: 'binary';
|
|
13
|
+
mediaType: string;
|
|
10
14
|
base64?: string;
|
|
11
15
|
}
|
|
16
|
+
export interface ExternalBinaryCloudflareManifestEntry {
|
|
17
|
+
path: string;
|
|
18
|
+
kind: 'binary';
|
|
19
|
+
mediaType: string;
|
|
20
|
+
storageKind: 'assets' | 'r2';
|
|
21
|
+
storageKey: string;
|
|
22
|
+
byteSize: number;
|
|
23
|
+
}
|
|
24
|
+
export type CloudflareManifestEntry = TextCloudflareManifestEntry | InlineBinaryCloudflareManifestEntry | ExternalBinaryCloudflareManifestEntry;
|
|
25
|
+
export interface CloudflareBundleRuntimeConfig {
|
|
26
|
+
binaryMode: 'inline' | 'external';
|
|
27
|
+
r2Binding?: string;
|
|
28
|
+
}
|
|
12
29
|
export interface CloudflareManifest {
|
|
13
30
|
entries: CloudflareManifestEntry[];
|
|
14
31
|
siteConfig?: ResolvedSiteConfig;
|
|
15
32
|
searchEntries?: SearchBundleEntry[];
|
|
33
|
+
externalSearchEntries?: ExternalSearchBundleEntry[];
|
|
34
|
+
runtime?: CloudflareBundleRuntimeConfig;
|
|
16
35
|
}
|
|
17
|
-
export interface
|
|
36
|
+
export interface CloudflareAssetsBindingLike {
|
|
18
37
|
fetch(request: Request): Promise<Response>;
|
|
19
38
|
}
|
|
39
|
+
export interface CloudflareR2ObjectBodyLike {
|
|
40
|
+
arrayBuffer(): Promise<ArrayBuffer>;
|
|
41
|
+
}
|
|
42
|
+
export interface CloudflareR2ObjectLike {
|
|
43
|
+
body: CloudflareR2ObjectBodyLike | ReadableStream | null;
|
|
44
|
+
arrayBuffer?: () => Promise<ArrayBuffer>;
|
|
45
|
+
httpEtag?: string;
|
|
46
|
+
}
|
|
47
|
+
export interface CloudflareR2BucketLike {
|
|
48
|
+
get(key: string): Promise<CloudflareR2ObjectLike | null>;
|
|
49
|
+
}
|
|
50
|
+
export interface CloudflareWorkerEnv {
|
|
51
|
+
ASSETS?: CloudflareAssetsBindingLike;
|
|
52
|
+
[binding: string]: unknown;
|
|
53
|
+
}
|
|
54
|
+
export interface ExportedHandlerLike {
|
|
55
|
+
fetch(request: Request, env?: CloudflareWorkerEnv, ctx?: unknown): Promise<Response>;
|
|
56
|
+
}
|
|
20
57
|
export interface CreateCloudflareWorkerOptions {
|
|
21
58
|
plugins?: MdoPlugin[];
|
|
22
59
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { MemoryContentStore
|
|
1
|
+
import { MemoryContentStore } from '../core/content-store.js';
|
|
2
2
|
import { handleSiteRequest } from '../core/request-handler.js';
|
|
3
|
-
import {
|
|
3
|
+
import { resolveRequest } from '../core/router.js';
|
|
4
|
+
import { createSearchApiFromBundle, createSearchApiFromExternalBundle, } from '../search.js';
|
|
4
5
|
export function createCloudflareWorker(manifest, options = {}) {
|
|
5
|
-
const
|
|
6
|
+
const storeIndex = new MemoryContentStore(manifest.entries.map((entry) => {
|
|
6
7
|
if (entry.kind === 'text') {
|
|
7
8
|
return {
|
|
8
9
|
path: entry.path,
|
|
@@ -11,6 +12,13 @@ export function createCloudflareWorker(manifest, options = {}) {
|
|
|
11
12
|
text: entry.text ?? '',
|
|
12
13
|
};
|
|
13
14
|
}
|
|
15
|
+
if ('storageKind' in entry) {
|
|
16
|
+
return {
|
|
17
|
+
path: entry.path,
|
|
18
|
+
kind: 'binary',
|
|
19
|
+
mediaType: entry.mediaType,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
14
22
|
return {
|
|
15
23
|
path: entry.path,
|
|
16
24
|
kind: 'binary',
|
|
@@ -18,12 +26,23 @@ export function createCloudflareWorker(manifest, options = {}) {
|
|
|
18
26
|
bytes: decodeBase64(entry.base64 ?? ''),
|
|
19
27
|
};
|
|
20
28
|
}));
|
|
21
|
-
const
|
|
29
|
+
const inlineSearchApi = manifest.searchEntries && manifest.searchEntries.length > 0
|
|
22
30
|
? createSearchApiFromBundle(manifest.searchEntries)
|
|
23
31
|
: undefined;
|
|
32
|
+
const externalSearchApis = new WeakMap();
|
|
33
|
+
let defaultExternalSearchApi;
|
|
24
34
|
return {
|
|
25
|
-
async fetch(request) {
|
|
35
|
+
async fetch(request, env) {
|
|
26
36
|
const url = new URL(request.url);
|
|
37
|
+
const externalSearchApi = getExternalSearchApi(manifest, env, externalSearchApis, defaultExternalSearchApi);
|
|
38
|
+
if (env === undefined && externalSearchApi !== undefined) {
|
|
39
|
+
defaultExternalSearchApi = externalSearchApi;
|
|
40
|
+
}
|
|
41
|
+
const directBinaryResponse = await tryServeExternalBinary(manifest, request, env);
|
|
42
|
+
if (directBinaryResponse !== null) {
|
|
43
|
+
return directBinaryResponse;
|
|
44
|
+
}
|
|
45
|
+
const store = new CloudflareManifestContentStore(manifest, storeIndex, request, env);
|
|
27
46
|
const siteResponse = await handleSiteRequest(store, url.pathname, {
|
|
28
47
|
draftMode: 'exclude',
|
|
29
48
|
siteConfig: manifest.siteConfig ?? {
|
|
@@ -33,23 +52,21 @@ export function createCloudflareWorker(manifest, options = {}) {
|
|
|
33
52
|
logo: undefined,
|
|
34
53
|
showDate: true,
|
|
35
54
|
showSummary: true,
|
|
36
|
-
theme: 'paper',
|
|
37
|
-
template: 'document',
|
|
38
55
|
topNav: [],
|
|
39
56
|
footerNav: [],
|
|
40
57
|
footerText: undefined,
|
|
41
58
|
socialLinks: [],
|
|
42
59
|
editLink: undefined,
|
|
43
60
|
showHomeIndex: true,
|
|
44
|
-
|
|
45
|
-
|
|
61
|
+
listingInitialPostCount: 10,
|
|
62
|
+
listingLoadMoreStep: 10,
|
|
46
63
|
siteTitleConfigured: false,
|
|
47
64
|
siteDescriptionConfigured: false,
|
|
48
65
|
},
|
|
49
66
|
acceptHeader: request.headers.get('accept') ?? undefined,
|
|
50
67
|
searchParams: url.searchParams,
|
|
51
68
|
requestUrl: request.url,
|
|
52
|
-
searchApi,
|
|
69
|
+
searchApi: inlineSearchApi ?? externalSearchApi,
|
|
53
70
|
plugins: options.plugins,
|
|
54
71
|
});
|
|
55
72
|
const headers = new Headers(siteResponse.headers);
|
|
@@ -65,6 +82,229 @@ export function createCloudflareWorker(manifest, options = {}) {
|
|
|
65
82
|
},
|
|
66
83
|
};
|
|
67
84
|
}
|
|
85
|
+
function getExternalSearchApi(manifest, env, cache, defaultApi) {
|
|
86
|
+
if (!manifest.externalSearchEntries || manifest.externalSearchEntries.length === 0) {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
if (!env) {
|
|
90
|
+
return (defaultApi ??
|
|
91
|
+
createSearchApiFromExternalBundle(manifest.externalSearchEntries, async (entry) => loadExternalSearchEntryResponse(entry, undefined, manifest.runtime?.r2Binding)));
|
|
92
|
+
}
|
|
93
|
+
const cached = cache.get(env);
|
|
94
|
+
if (cached) {
|
|
95
|
+
return cached;
|
|
96
|
+
}
|
|
97
|
+
const searchApi = createSearchApiFromExternalBundle(manifest.externalSearchEntries, async (entry) => loadExternalSearchEntryResponse(entry, env, manifest.runtime?.r2Binding));
|
|
98
|
+
cache.set(env, searchApi);
|
|
99
|
+
return searchApi;
|
|
100
|
+
}
|
|
101
|
+
async function loadExternalSearchEntryResponse(entry, env, r2Binding) {
|
|
102
|
+
if (entry.storageKind === 'assets') {
|
|
103
|
+
const assetsBinding = env?.ASSETS;
|
|
104
|
+
if (!assetsBinding) {
|
|
105
|
+
throw new Error(`Cloudflare ASSETS binding is required to serve search bundle file ${entry.path}.`);
|
|
106
|
+
}
|
|
107
|
+
const assetResponse = await assetsBinding.fetch(new Request(new URL(`/${entry.storageKey}`, 'https://mdorigin-search.invalid/'), {
|
|
108
|
+
method: 'GET',
|
|
109
|
+
}));
|
|
110
|
+
if (assetResponse.ok) {
|
|
111
|
+
return assetResponse;
|
|
112
|
+
}
|
|
113
|
+
return new Response('Not Found', {
|
|
114
|
+
status: 404,
|
|
115
|
+
headers: {
|
|
116
|
+
'content-type': 'text/plain; charset=utf-8',
|
|
117
|
+
},
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
const bindingName = r2Binding ?? 'MDORIGIN_R2';
|
|
121
|
+
const bucket = env?.[bindingName];
|
|
122
|
+
if (!bucket) {
|
|
123
|
+
throw new Error(`Cloudflare R2 binding ${bindingName} is required to serve search bundle file ${entry.path}.`);
|
|
124
|
+
}
|
|
125
|
+
const object = await bucket.get(entry.storageKey);
|
|
126
|
+
if (!object) {
|
|
127
|
+
return new Response('Not Found', {
|
|
128
|
+
status: 404,
|
|
129
|
+
headers: {
|
|
130
|
+
'content-type': 'text/plain; charset=utf-8',
|
|
131
|
+
},
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
const headers = new Headers({
|
|
135
|
+
'content-type': entry.mediaType,
|
|
136
|
+
});
|
|
137
|
+
if (object.httpEtag) {
|
|
138
|
+
headers.set('etag', object.httpEtag);
|
|
139
|
+
}
|
|
140
|
+
if (object.body instanceof ReadableStream) {
|
|
141
|
+
return new Response(object.body, {
|
|
142
|
+
status: 200,
|
|
143
|
+
headers,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
if (object.body && 'arrayBuffer' in object.body) {
|
|
147
|
+
return new Response(await object.body.arrayBuffer(), {
|
|
148
|
+
status: 200,
|
|
149
|
+
headers,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
if (typeof object.arrayBuffer === 'function') {
|
|
153
|
+
return new Response(await object.arrayBuffer(), {
|
|
154
|
+
status: 200,
|
|
155
|
+
headers,
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
return new Response(null, {
|
|
159
|
+
status: 200,
|
|
160
|
+
headers,
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
async function tryServeExternalBinary(manifest, request, env) {
|
|
164
|
+
const resolved = resolveRequest(new URL(request.url).pathname);
|
|
165
|
+
if (resolved.kind !== 'asset' || !resolved.sourcePath) {
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
const manifestEntry = manifest.entries.find((entry) => entry.path === resolved.sourcePath && isExternalBinaryEntry(entry));
|
|
169
|
+
if (!manifestEntry) {
|
|
170
|
+
return null;
|
|
171
|
+
}
|
|
172
|
+
if (manifestEntry.storageKind === 'assets') {
|
|
173
|
+
const assetsBinding = env?.ASSETS;
|
|
174
|
+
if (!assetsBinding) {
|
|
175
|
+
throw new Error(`Cloudflare ASSETS binding is required to serve ${manifestEntry.path}.`);
|
|
176
|
+
}
|
|
177
|
+
const assetUrl = new URL(request.url);
|
|
178
|
+
assetUrl.pathname = `/${manifestEntry.storageKey}`;
|
|
179
|
+
return assetsBinding.fetch(new Request(assetUrl.toString(), request));
|
|
180
|
+
}
|
|
181
|
+
const bucket = env?.[manifest.runtime?.r2Binding ?? 'MDORIGIN_R2'];
|
|
182
|
+
if (!bucket) {
|
|
183
|
+
throw new Error(`Cloudflare R2 binding ${manifest.runtime?.r2Binding ?? 'MDORIGIN_R2'} is required to serve ${manifestEntry.path}.`);
|
|
184
|
+
}
|
|
185
|
+
const object = await bucket.get(manifestEntry.storageKey);
|
|
186
|
+
if (!object) {
|
|
187
|
+
return new Response('Not Found', {
|
|
188
|
+
status: 404,
|
|
189
|
+
headers: {
|
|
190
|
+
'content-type': 'text/plain; charset=utf-8',
|
|
191
|
+
},
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
const headers = new Headers({
|
|
195
|
+
'content-type': manifestEntry.mediaType,
|
|
196
|
+
});
|
|
197
|
+
if (object.httpEtag) {
|
|
198
|
+
headers.set('etag', object.httpEtag);
|
|
199
|
+
}
|
|
200
|
+
if (request.method === 'HEAD') {
|
|
201
|
+
return new Response(null, {
|
|
202
|
+
status: 200,
|
|
203
|
+
headers,
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
if (object.body instanceof ReadableStream) {
|
|
207
|
+
return new Response(object.body, {
|
|
208
|
+
status: 200,
|
|
209
|
+
headers,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
if (object.body && 'arrayBuffer' in object.body) {
|
|
213
|
+
return new Response(await object.body.arrayBuffer(), {
|
|
214
|
+
status: 200,
|
|
215
|
+
headers,
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
if (typeof object.arrayBuffer === 'function') {
|
|
219
|
+
return new Response(await object.arrayBuffer(), {
|
|
220
|
+
status: 200,
|
|
221
|
+
headers,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
return new Response(null, {
|
|
225
|
+
status: 200,
|
|
226
|
+
headers,
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
function isExternalBinaryEntry(entry) {
|
|
230
|
+
return entry.kind === 'binary' && 'storageKind' in entry;
|
|
231
|
+
}
|
|
232
|
+
class CloudflareManifestContentStore {
|
|
233
|
+
storeIndex;
|
|
234
|
+
request;
|
|
235
|
+
env;
|
|
236
|
+
entries;
|
|
237
|
+
runtime;
|
|
238
|
+
constructor(manifest, storeIndex, request, env) {
|
|
239
|
+
this.storeIndex = storeIndex;
|
|
240
|
+
this.request = request;
|
|
241
|
+
this.env = env;
|
|
242
|
+
this.entries = new Map(manifest.entries.map((entry) => [entry.path, entry]));
|
|
243
|
+
this.runtime = manifest.runtime;
|
|
244
|
+
}
|
|
245
|
+
async get(contentPath) {
|
|
246
|
+
const baseEntry = await this.storeIndex.get(contentPath);
|
|
247
|
+
if (baseEntry === null) {
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
const manifestEntry = this.entries.get(contentPath);
|
|
251
|
+
if (!manifestEntry || manifestEntry.kind === 'text') {
|
|
252
|
+
return baseEntry;
|
|
253
|
+
}
|
|
254
|
+
if (!('storageKind' in manifestEntry)) {
|
|
255
|
+
return baseEntry;
|
|
256
|
+
}
|
|
257
|
+
if (manifestEntry.storageKind === 'assets') {
|
|
258
|
+
const assetsBinding = this.env?.ASSETS;
|
|
259
|
+
if (!assetsBinding) {
|
|
260
|
+
throw new Error(`Cloudflare ASSETS binding is required to serve ${manifestEntry.path}.`);
|
|
261
|
+
}
|
|
262
|
+
const assetResponse = await assetsBinding.fetch(new Request(new URL(`/${manifestEntry.storageKey}`, this.request.url), {
|
|
263
|
+
method: 'GET',
|
|
264
|
+
}));
|
|
265
|
+
if (!assetResponse.ok) {
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
268
|
+
return {
|
|
269
|
+
path: manifestEntry.path,
|
|
270
|
+
kind: 'binary',
|
|
271
|
+
mediaType: manifestEntry.mediaType,
|
|
272
|
+
bytes: new Uint8Array(await assetResponse.arrayBuffer()),
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
const r2BindingName = this.findR2BindingName(manifestEntry);
|
|
276
|
+
const bucket = this.env?.[r2BindingName];
|
|
277
|
+
if (!bucket) {
|
|
278
|
+
throw new Error(`Cloudflare R2 binding ${r2BindingName} is required to serve ${manifestEntry.path}.`);
|
|
279
|
+
}
|
|
280
|
+
const object = await bucket.get(manifestEntry.storageKey);
|
|
281
|
+
if (!object) {
|
|
282
|
+
return null;
|
|
283
|
+
}
|
|
284
|
+
const arrayBuffer = typeof object.arrayBuffer === 'function'
|
|
285
|
+
? await object.arrayBuffer()
|
|
286
|
+
: object.body instanceof ReadableStream
|
|
287
|
+
? await new Response(object.body).arrayBuffer()
|
|
288
|
+
: object.body && 'arrayBuffer' in object.body
|
|
289
|
+
? await object.body.arrayBuffer()
|
|
290
|
+
: new ArrayBuffer(0);
|
|
291
|
+
return {
|
|
292
|
+
path: manifestEntry.path,
|
|
293
|
+
kind: 'binary',
|
|
294
|
+
mediaType: manifestEntry.mediaType,
|
|
295
|
+
bytes: new Uint8Array(arrayBuffer),
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
async listDirectory(contentPath) {
|
|
299
|
+
return this.storeIndex.listDirectory(contentPath);
|
|
300
|
+
}
|
|
301
|
+
findR2BindingName(entry) {
|
|
302
|
+
if (entry.storageKind !== 'r2') {
|
|
303
|
+
return 'ASSETS';
|
|
304
|
+
}
|
|
305
|
+
return this.runtime?.r2Binding ?? 'MDORIGIN_R2';
|
|
306
|
+
}
|
|
307
|
+
}
|
|
68
308
|
function decodeBase64(value) {
|
|
69
309
|
const decoded = atob(value);
|
|
70
310
|
return Uint8Array.from(decoded, (character) => character.charCodeAt(0));
|
package/dist/adapters/node.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createServer } from 'node:http';
|
|
2
2
|
import { readFile, readdir, stat } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
|
-
import { getMediaTypeForPath, isLikelyTextPath, normalizeContentPath, normalizeDirectoryPath, } from '../core/content-store.js';
|
|
4
|
+
import { getMediaTypeForPath, isIgnoredContentName, isLikelyTextPath, normalizeContentPath, normalizeDirectoryPath, } from '../core/content-store.js';
|
|
5
5
|
import { handleSiteRequest } from '../core/request-handler.js';
|
|
6
6
|
export function createFileSystemContentStore(rootDir) {
|
|
7
7
|
const resolvedRootDir = path.resolve(rootDir);
|
|
@@ -61,7 +61,7 @@ export function createFileSystemContentStore(rootDir) {
|
|
|
61
61
|
}
|
|
62
62
|
const entries = await readdir(directoryPath, { withFileTypes: true });
|
|
63
63
|
const resolvedEntries = await Promise.all(entries
|
|
64
|
-
.filter((entry) => !entry.name
|
|
64
|
+
.filter((entry) => !isIgnoredContentName(entry.name))
|
|
65
65
|
.map(async (entry) => {
|
|
66
66
|
const childVisiblePath = normalizedPath === ''
|
|
67
67
|
? entry.name
|
|
@@ -5,11 +5,11 @@ import { applySiteConfigFrontmatterDefaults, loadUserSiteConfig, } from '../core
|
|
|
5
5
|
export async function runBuildCloudflareCommand(argv) {
|
|
6
6
|
const args = parseArgs(argv);
|
|
7
7
|
if (args.help) {
|
|
8
|
-
console.log('Usage: mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config <config-file>] [--search ./dist/search]');
|
|
8
|
+
console.log('Usage: mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config <config-file>] [--search ./dist/search] [--binary-mode inline|external] [--assets-max-bytes 26214400] [--r2-binding MDORIGIN_R2]');
|
|
9
9
|
return;
|
|
10
10
|
}
|
|
11
11
|
if (!args.root) {
|
|
12
|
-
console.error('Usage: mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config <config-file>] [--search ./dist/search]');
|
|
12
|
+
console.error('Usage: mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config <config-file>] [--search ./dist/search] [--binary-mode inline|external] [--assets-max-bytes 26214400] [--r2-binding MDORIGIN_R2]');
|
|
13
13
|
process.exitCode = 1;
|
|
14
14
|
return;
|
|
15
15
|
}
|
|
@@ -27,6 +27,9 @@ export async function runBuildCloudflareCommand(argv) {
|
|
|
27
27
|
siteConfig,
|
|
28
28
|
searchDir: args.search ? path.resolve(args.search) : undefined,
|
|
29
29
|
configModulePath: loadedConfig.configModulePath,
|
|
30
|
+
binaryMode: args.binaryMode,
|
|
31
|
+
assetsMaxBytes: args.assetsMaxBytes,
|
|
32
|
+
r2Binding: args.r2Binding,
|
|
30
33
|
});
|
|
31
34
|
console.log(`cloudflare worker written to ${result.workerFile}`);
|
|
32
35
|
}
|
|
@@ -59,6 +62,28 @@ function parseArgs(argv) {
|
|
|
59
62
|
index += 1;
|
|
60
63
|
continue;
|
|
61
64
|
}
|
|
65
|
+
if (argument === '--binary-mode' && nextValue) {
|
|
66
|
+
if (nextValue !== 'inline' && nextValue !== 'external') {
|
|
67
|
+
throw new Error(`Invalid binary mode for mdorigin build cloudflare: ${nextValue}`);
|
|
68
|
+
}
|
|
69
|
+
result.binaryMode = nextValue;
|
|
70
|
+
index += 1;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (argument === '--assets-max-bytes' && nextValue) {
|
|
74
|
+
const parsed = Number.parseInt(nextValue, 10);
|
|
75
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
76
|
+
throw new Error(`Invalid value for --assets-max-bytes: ${nextValue}`);
|
|
77
|
+
}
|
|
78
|
+
result.assetsMaxBytes = parsed;
|
|
79
|
+
index += 1;
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
if (argument === '--r2-binding' && nextValue) {
|
|
83
|
+
result.r2Binding = nextValue;
|
|
84
|
+
index += 1;
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
62
87
|
throw new Error(`Unknown argument for mdorigin build cloudflare: ${argument}`);
|
|
63
88
|
}
|
|
64
89
|
return result;
|
package/dist/cli/build-search.js
CHANGED
|
@@ -5,11 +5,11 @@ import { createFileSystemContentStore } from '../adapters/node.js';
|
|
|
5
5
|
export async function runBuildSearchCommand(rawArgs) {
|
|
6
6
|
const args = parseArgs(rawArgs);
|
|
7
7
|
if (args.help) {
|
|
8
|
-
console.log('Usage: mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config <config-file>]');
|
|
8
|
+
console.log('Usage: mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config <config-file>] [--incremental]');
|
|
9
9
|
return;
|
|
10
10
|
}
|
|
11
11
|
if (!args.root) {
|
|
12
|
-
throw new Error('Usage: mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config <config-file>]');
|
|
12
|
+
throw new Error('Usage: mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config <config-file>] [--incremental]');
|
|
13
13
|
}
|
|
14
14
|
const rootDir = path.resolve(args.root);
|
|
15
15
|
const store = createFileSystemContentStore(rootDir);
|
|
@@ -24,18 +24,28 @@ export async function runBuildSearchCommand(rawArgs) {
|
|
|
24
24
|
siteConfig,
|
|
25
25
|
embeddingBackend: args.embeddingBackend ?? 'model2vec',
|
|
26
26
|
model: args.model,
|
|
27
|
+
incremental: args.incremental,
|
|
27
28
|
});
|
|
28
|
-
console.log(
|
|
29
|
+
console.log([
|
|
30
|
+
`search bundle written to ${result.outputDir} (${result.documentCount} documents, ${result.chunkCount} chunks)`,
|
|
31
|
+
result.incremental
|
|
32
|
+
? `incremental cache: ${result.cachePath} (${result.incremental.newDocumentCount} new, ${result.incremental.changedDocumentCount} changed, ${result.incremental.unchangedDocumentCount} unchanged, ${result.incremental.removedDocumentCount} removed)`
|
|
33
|
+
: '',
|
|
34
|
+
].filter(Boolean).join('\n'));
|
|
29
35
|
}
|
|
30
36
|
function parseArgs(rawArgs) {
|
|
31
37
|
const parsed = {};
|
|
32
|
-
const supportedFlags = new Set(['root', 'out', 'embedding-backend', 'model', 'config']);
|
|
38
|
+
const supportedFlags = new Set(['root', 'out', 'embedding-backend', 'model', 'config', 'incremental']);
|
|
33
39
|
for (let index = 0; index < rawArgs.length; index += 1) {
|
|
34
40
|
const arg = rawArgs[index];
|
|
35
41
|
if (arg === '--help' || arg === '-h') {
|
|
36
42
|
parsed.help = 'true';
|
|
37
43
|
continue;
|
|
38
44
|
}
|
|
45
|
+
if (arg === '--incremental') {
|
|
46
|
+
parsed.incremental = 'true';
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
39
49
|
if (arg.startsWith('--')) {
|
|
40
50
|
const flag = arg.slice(2);
|
|
41
51
|
if (!supportedFlags.has(flag)) {
|
|
@@ -57,6 +67,7 @@ function parseArgs(rawArgs) {
|
|
|
57
67
|
embeddingBackend: parsed['embedding-backend'],
|
|
58
68
|
model: parsed.model,
|
|
59
69
|
config: parsed.config,
|
|
70
|
+
incremental: parsed.incremental === 'true',
|
|
60
71
|
help: parsed.help === 'true',
|
|
61
72
|
};
|
|
62
73
|
}
|
package/dist/cli/help.js
CHANGED
|
@@ -2,10 +2,11 @@ export const ROOT_USAGE_LINES = [
|
|
|
2
2
|
'Usage:',
|
|
3
3
|
' mdorigin dev --root <content-dir> [--port 3000] [--config <config-file>] [--search ./dist/search]',
|
|
4
4
|
' mdorigin build index (--root <content-dir> | --dir <content-dir>) [--config <config-file>]',
|
|
5
|
-
' mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config <config-file>]',
|
|
6
|
-
' mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config <config-file>] [--search ./dist/search]',
|
|
7
|
-
' mdorigin init cloudflare [--dir .] [--entry ./dist/cloudflare/worker.mjs] [--name <worker-name>] [--compatibility-date 2026-03-20] [--force]',
|
|
8
|
-
' mdorigin
|
|
5
|
+
' mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config <config-file>] [--incremental]',
|
|
6
|
+
' mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config <config-file>] [--search ./dist/search] [--binary-mode inline|external] [--assets-max-bytes 26214400] [--r2-binding MDORIGIN_R2]',
|
|
7
|
+
' mdorigin init cloudflare [--dir .] [--entry ./dist/cloudflare/worker.mjs] [--name <worker-name>] [--compatibility-date 2026-03-20] [--r2-bucket <bucket-name>] [--force]',
|
|
8
|
+
' mdorigin sync cloudflare-r2 --dir ./dist/cloudflare --bucket <bucket-name> [--force]',
|
|
9
|
+
' mdorigin search --index <search-dir> [--top-k 10] [--meta key=value] <query>',
|
|
9
10
|
' mdorigin version',
|
|
10
11
|
'',
|
|
11
12
|
'Global options:',
|
|
@@ -15,12 +16,12 @@ export const ROOT_USAGE_LINES = [
|
|
|
15
16
|
export const BUILD_USAGE_LINES = [
|
|
16
17
|
'Usage:',
|
|
17
18
|
' mdorigin build index (--root <content-dir> | --dir <content-dir>) [--config <config-file>]',
|
|
18
|
-
' mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config <config-file>]',
|
|
19
|
-
' mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config <config-file>] [--search ./dist/search]',
|
|
19
|
+
' mdorigin build search --root <content-dir> [--out ./dist/search] [--embedding-backend model2vec|hashing] [--model sentence-transformers/all-MiniLM-L6-v2] [--config <config-file>] [--incremental]',
|
|
20
|
+
' mdorigin build cloudflare --root <content-dir> [--out ./dist/cloudflare] [--config <config-file>] [--search ./dist/search] [--binary-mode inline|external] [--assets-max-bytes 26214400] [--r2-binding MDORIGIN_R2]',
|
|
20
21
|
];
|
|
21
22
|
export const INIT_USAGE_LINES = [
|
|
22
23
|
'Usage:',
|
|
23
|
-
' mdorigin init cloudflare [--dir .] [--entry ./dist/cloudflare/worker.mjs] [--name <worker-name>] [--compatibility-date 2026-03-20] [--force]',
|
|
24
|
+
' mdorigin init cloudflare [--dir .] [--entry ./dist/cloudflare/worker.mjs] [--name <worker-name>] [--compatibility-date 2026-03-20] [--r2-bucket <bucket-name>] [--force]',
|
|
24
25
|
];
|
|
25
26
|
export function printUsage(lines, error = false) {
|
|
26
27
|
const output = lines.join('\n');
|
|
@@ -4,7 +4,7 @@ import { initCloudflareProject } from '../cloudflare.js';
|
|
|
4
4
|
export async function runInitCloudflareCommand(argv) {
|
|
5
5
|
const args = parseArgs(argv);
|
|
6
6
|
if (args.help) {
|
|
7
|
-
console.log('Usage: mdorigin init cloudflare [--dir .] [--entry ./dist/cloudflare/worker.mjs] [--name <worker-name>] [--compatibility-date 2026-03-20] [--force]');
|
|
7
|
+
console.log('Usage: mdorigin init cloudflare [--dir .] [--entry ./dist/cloudflare/worker.mjs] [--name <worker-name>] [--compatibility-date 2026-03-20] [--r2-bucket <bucket-name>] [--force]');
|
|
8
8
|
return;
|
|
9
9
|
}
|
|
10
10
|
const projectDir = path.resolve(args.dir ?? '.');
|
|
@@ -16,6 +16,7 @@ export async function runInitCloudflareCommand(argv) {
|
|
|
16
16
|
workerName: args.name,
|
|
17
17
|
siteTitle,
|
|
18
18
|
compatibilityDate: args.compatibilityDate,
|
|
19
|
+
r2Bucket: args.r2Bucket,
|
|
19
20
|
force: args.force,
|
|
20
21
|
});
|
|
21
22
|
console.log(`wrangler config written to ${result.configFile}`);
|
|
@@ -49,6 +50,11 @@ function parseArgs(argv) {
|
|
|
49
50
|
index += 1;
|
|
50
51
|
continue;
|
|
51
52
|
}
|
|
53
|
+
if (argument === '--r2-bucket' && nextValue) {
|
|
54
|
+
result.r2Bucket = nextValue;
|
|
55
|
+
index += 1;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
52
58
|
if (argument === '--force') {
|
|
53
59
|
result.force = true;
|
|
54
60
|
continue;
|
package/dist/cli/main.js
CHANGED
|
@@ -9,6 +9,7 @@ import { runDevCommand } from './dev.js';
|
|
|
9
9
|
import { BUILD_USAGE_LINES, INIT_USAGE_LINES, ROOT_USAGE_LINES, printUsage } from './help.js';
|
|
10
10
|
import { runInitCloudflareCommand } from './init-cloudflare.js';
|
|
11
11
|
import { runSearchCommand } from './search.js';
|
|
12
|
+
import { runSyncCloudflareR2Command } from './sync-cloudflare-r2.js';
|
|
12
13
|
async function main() {
|
|
13
14
|
const argv = process.argv.slice(2);
|
|
14
15
|
const [command, subcommand, ...rest] = argv;
|
|
@@ -62,6 +63,10 @@ async function main() {
|
|
|
62
63
|
await runSearchCommand([subcommand, ...rest].filter(isDefined));
|
|
63
64
|
return;
|
|
64
65
|
}
|
|
66
|
+
if (command === 'sync' && subcommand === 'cloudflare-r2') {
|
|
67
|
+
await runSyncCloudflareR2Command(rest);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
65
70
|
console.error(`Unknown command: ${argv.join(' ')}`);
|
|
66
71
|
printUsage(ROOT_USAGE_LINES, true);
|
|
67
72
|
process.exitCode = 1;
|
package/dist/cli/search.js
CHANGED
|
@@ -3,23 +3,25 @@ import { searchBundle } from '../search.js';
|
|
|
3
3
|
export async function runSearchCommand(rawArgs) {
|
|
4
4
|
const args = parseArgs(rawArgs);
|
|
5
5
|
if (args.help) {
|
|
6
|
-
console.log('Usage: mdorigin search --index <search-dir> [--top-k 10] <query>');
|
|
6
|
+
console.log('Usage: mdorigin search --index <search-dir> [--top-k 10] [--meta key=value] <query>');
|
|
7
7
|
return;
|
|
8
8
|
}
|
|
9
9
|
if (!args.indexDir || !args.query) {
|
|
10
|
-
throw new Error('Usage: mdorigin search --index <search-dir> [--top-k 10] <query>');
|
|
10
|
+
throw new Error('Usage: mdorigin search --index <search-dir> [--top-k 10] [--meta key=value] <query>');
|
|
11
11
|
}
|
|
12
12
|
const hits = await searchBundle({
|
|
13
13
|
indexDir: path.resolve(args.indexDir),
|
|
14
14
|
query: args.query,
|
|
15
15
|
topK: args.topK,
|
|
16
|
+
metadata: args.metadata,
|
|
16
17
|
});
|
|
17
18
|
console.log(JSON.stringify(hits, null, 2));
|
|
18
19
|
}
|
|
19
20
|
function parseArgs(rawArgs) {
|
|
20
21
|
const flags = {};
|
|
22
|
+
const metadata = {};
|
|
21
23
|
const positionals = [];
|
|
22
|
-
const supportedFlags = new Set(['index', 'top-k']);
|
|
24
|
+
const supportedFlags = new Set(['index', 'top-k', 'meta']);
|
|
23
25
|
for (let index = 0; index < rawArgs.length; index += 1) {
|
|
24
26
|
const arg = rawArgs[index];
|
|
25
27
|
if (arg === '--help' || arg === '-h') {
|
|
@@ -33,7 +35,16 @@ function parseArgs(rawArgs) {
|
|
|
33
35
|
}
|
|
34
36
|
const value = rawArgs[index + 1];
|
|
35
37
|
if (value && !value.startsWith('--')) {
|
|
36
|
-
|
|
38
|
+
if (flag === 'meta') {
|
|
39
|
+
const separator = value.indexOf('=');
|
|
40
|
+
if (separator <= 0 || separator === value.length - 1) {
|
|
41
|
+
throw new Error(`Invalid value for --meta: ${value}`);
|
|
42
|
+
}
|
|
43
|
+
metadata[value.slice(0, separator)] = value.slice(separator + 1);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
flags[flag] = value;
|
|
47
|
+
}
|
|
37
48
|
index += 1;
|
|
38
49
|
continue;
|
|
39
50
|
}
|
|
@@ -46,6 +57,7 @@ function parseArgs(rawArgs) {
|
|
|
46
57
|
indexDir: flags.index,
|
|
47
58
|
topK: Number.isInteger(topK) && topK > 0 ? topK : undefined,
|
|
48
59
|
query: positionals.join(' ').trim(),
|
|
60
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
|
|
49
61
|
help: flags.help === 'true',
|
|
50
62
|
};
|
|
51
63
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function runSyncCloudflareR2Command(argv: string[]): Promise<void>;
|