mdorigin 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/cloudflare.js +3 -3
- package/dist/cli/dev.js +1 -1
- package/dist/core/api.js +4 -2
- package/dist/core/site-config.d.ts +33 -0
- package/dist/core/site-config.js +166 -0
- package/dist/search.d.ts +12 -4
- package/dist/search.js +152 -104
- package/package.json +2 -2
|
@@ -27,7 +27,7 @@ export function createCloudflareWorker(manifest, options = {}) {
|
|
|
27
27
|
};
|
|
28
28
|
}));
|
|
29
29
|
const inlineSearchApi = manifest.searchEntries && manifest.searchEntries.length > 0
|
|
30
|
-
? createSearchApiFromBundle(manifest.searchEntries)
|
|
30
|
+
? createSearchApiFromBundle(manifest.searchEntries, manifest.siteConfig?.search)
|
|
31
31
|
: undefined;
|
|
32
32
|
const externalSearchApis = new WeakMap();
|
|
33
33
|
let defaultExternalSearchApi;
|
|
@@ -88,13 +88,13 @@ function getExternalSearchApi(manifest, env, cache, defaultApi) {
|
|
|
88
88
|
}
|
|
89
89
|
if (!env) {
|
|
90
90
|
return (defaultApi ??
|
|
91
|
-
createSearchApiFromExternalBundle(manifest.externalSearchEntries, async (entry) => loadExternalSearchEntryResponse(entry, undefined, manifest.runtime?.r2Binding)));
|
|
91
|
+
createSearchApiFromExternalBundle(manifest.externalSearchEntries, async (entry) => loadExternalSearchEntryResponse(entry, undefined, manifest.runtime?.r2Binding), manifest.siteConfig?.search));
|
|
92
92
|
}
|
|
93
93
|
const cached = cache.get(env);
|
|
94
94
|
if (cached) {
|
|
95
95
|
return cached;
|
|
96
96
|
}
|
|
97
|
-
const searchApi = createSearchApiFromExternalBundle(manifest.externalSearchEntries, async (entry) => loadExternalSearchEntryResponse(entry, env, manifest.runtime?.r2Binding));
|
|
97
|
+
const searchApi = createSearchApiFromExternalBundle(manifest.externalSearchEntries, async (entry) => loadExternalSearchEntryResponse(entry, env, manifest.runtime?.r2Binding), manifest.siteConfig?.search);
|
|
98
98
|
cache.set(env, searchApi);
|
|
99
99
|
return searchApi;
|
|
100
100
|
}
|
package/dist/cli/dev.js
CHANGED
|
@@ -27,7 +27,7 @@ export async function runDevCommand(argv) {
|
|
|
27
27
|
draftMode: 'include',
|
|
28
28
|
siteConfig,
|
|
29
29
|
searchApi: args.search
|
|
30
|
-
? await createSearchApiFromDirectory(path.resolve(args.search))
|
|
30
|
+
? await createSearchApiFromDirectory(path.resolve(args.search), siteConfig.search)
|
|
31
31
|
: undefined,
|
|
32
32
|
plugins: loadedConfig.plugins,
|
|
33
33
|
});
|
package/dist/core/api.js
CHANGED
|
@@ -12,7 +12,9 @@ export async function handleApiRoute(pathname, searchParams, options) {
|
|
|
12
12
|
error: 'missing required query parameter: q',
|
|
13
13
|
});
|
|
14
14
|
}
|
|
15
|
-
const topK = normalizePositiveInteger(searchParams?.get('topK')) ??
|
|
15
|
+
const topK = normalizePositiveInteger(searchParams?.get('topK')) ??
|
|
16
|
+
options.siteConfig.search?.topK ??
|
|
17
|
+
10;
|
|
16
18
|
const metadata = readSearchMetadataFilters(searchParams);
|
|
17
19
|
const hits = await options.searchApi.search(query, {
|
|
18
20
|
topK,
|
|
@@ -66,7 +68,7 @@ function buildOpenApiDocument(options) {
|
|
|
66
68
|
in: 'query',
|
|
67
69
|
required: false,
|
|
68
70
|
schema: { type: 'string' },
|
|
69
|
-
description: 'Exact-match metadata filter. Use query parameters such as meta.type=post or meta.section=guides.',
|
|
71
|
+
description: 'Exact-match metadata filter. Use query parameters such as meta.type=post or meta.section=guides. Retrieval mode and reranking stay under site configuration.',
|
|
70
72
|
},
|
|
71
73
|
],
|
|
72
74
|
responses: {
|
|
@@ -17,6 +17,37 @@ export interface SiteSocialLink {
|
|
|
17
17
|
export interface EditLinkConfig {
|
|
18
18
|
baseUrl: string;
|
|
19
19
|
}
|
|
20
|
+
export interface SiteSearchRerankerConfig {
|
|
21
|
+
kind?: 'embedding-v1' | 'heuristic-v1';
|
|
22
|
+
candidatePoolSize?: number;
|
|
23
|
+
}
|
|
24
|
+
export interface SiteSearchScoreAdjustmentConfig {
|
|
25
|
+
metadataNumericMultiplier?: string;
|
|
26
|
+
}
|
|
27
|
+
export interface SiteSearchPolicyOverrideConfig {
|
|
28
|
+
mode?: 'hybrid' | 'vector' | null;
|
|
29
|
+
minScore?: number | null;
|
|
30
|
+
reranker?: SiteSearchRerankerConfig | null;
|
|
31
|
+
scoreAdjustment?: SiteSearchScoreAdjustmentConfig | null;
|
|
32
|
+
}
|
|
33
|
+
export interface SiteSearchShortQueryPolicyConfig extends SiteSearchPolicyOverrideConfig {
|
|
34
|
+
maxChars: number;
|
|
35
|
+
}
|
|
36
|
+
export interface SiteSearchLongQueryPolicyConfig extends SiteSearchPolicyOverrideConfig {
|
|
37
|
+
minChars: number;
|
|
38
|
+
}
|
|
39
|
+
export interface SiteSearchPolicyConfig {
|
|
40
|
+
shortQuery?: SiteSearchShortQueryPolicyConfig;
|
|
41
|
+
longQuery?: SiteSearchLongQueryPolicyConfig;
|
|
42
|
+
}
|
|
43
|
+
export interface SiteSearchConfig {
|
|
44
|
+
topK?: number;
|
|
45
|
+
mode?: 'hybrid' | 'vector';
|
|
46
|
+
minScore?: number;
|
|
47
|
+
reranker?: SiteSearchRerankerConfig;
|
|
48
|
+
scoreAdjustment?: SiteSearchScoreAdjustmentConfig;
|
|
49
|
+
policy?: SiteSearchPolicyConfig;
|
|
50
|
+
}
|
|
20
51
|
export interface SiteConfig {
|
|
21
52
|
siteTitle?: string;
|
|
22
53
|
siteDescription?: string;
|
|
@@ -35,6 +66,7 @@ export interface SiteConfig {
|
|
|
35
66
|
showHomeIndex?: boolean;
|
|
36
67
|
listingInitialPostCount?: number;
|
|
37
68
|
listingLoadMoreStep?: number;
|
|
69
|
+
search?: SiteSearchConfig;
|
|
38
70
|
}
|
|
39
71
|
export interface UserSiteConfig extends SiteConfig {
|
|
40
72
|
plugins?: MdoPlugin[];
|
|
@@ -56,6 +88,7 @@ export interface ResolvedSiteConfig {
|
|
|
56
88
|
showHomeIndex: boolean;
|
|
57
89
|
listingInitialPostCount: number;
|
|
58
90
|
listingLoadMoreStep: number;
|
|
91
|
+
search?: SiteSearchConfig;
|
|
59
92
|
stylesheetContent?: string;
|
|
60
93
|
siteTitleConfigured: boolean;
|
|
61
94
|
siteDescriptionConfigured: boolean;
|
package/dist/core/site-config.js
CHANGED
|
@@ -47,6 +47,7 @@ export async function loadUserSiteConfig(options = {}) {
|
|
|
47
47
|
: normalizeTopNav(parsedConfig.topNav).length === 0,
|
|
48
48
|
listingInitialPostCount: normalizePositiveInteger(parsedConfig.listingInitialPostCount ?? legacyConfig.catalogInitialPostCount, 10),
|
|
49
49
|
listingLoadMoreStep: normalizePositiveInteger(parsedConfig.listingLoadMoreStep ?? legacyConfig.catalogLoadMoreStep, 10),
|
|
50
|
+
search: normalizeSearchConfig(parsedConfig.search, configFilePath),
|
|
50
51
|
stylesheetContent,
|
|
51
52
|
siteTitleConfigured: typeof parsedConfig.siteTitle === 'string' && parsedConfig.siteTitle !== '',
|
|
52
53
|
siteDescriptionConfigured: typeof parsedConfig.siteDescription === 'string' &&
|
|
@@ -197,6 +198,171 @@ function normalizePositiveInteger(value, fallback) {
|
|
|
197
198
|
}
|
|
198
199
|
return fallback;
|
|
199
200
|
}
|
|
201
|
+
function normalizeOptionalPositiveInteger(value) {
|
|
202
|
+
if (typeof value === 'number' && Number.isInteger(value) && value > 0) {
|
|
203
|
+
return value;
|
|
204
|
+
}
|
|
205
|
+
if (typeof value === 'string') {
|
|
206
|
+
const parsed = Number.parseInt(value, 10);
|
|
207
|
+
if (Number.isInteger(parsed) && parsed > 0) {
|
|
208
|
+
return parsed;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return undefined;
|
|
212
|
+
}
|
|
213
|
+
function normalizeOptionalNumber(value) {
|
|
214
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
215
|
+
return value;
|
|
216
|
+
}
|
|
217
|
+
if (typeof value === 'string') {
|
|
218
|
+
const parsed = Number.parseFloat(value);
|
|
219
|
+
if (Number.isFinite(parsed)) {
|
|
220
|
+
return parsed;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return undefined;
|
|
224
|
+
}
|
|
225
|
+
function normalizeSearchConfig(value, configFilePath) {
|
|
226
|
+
if (typeof value !== 'object' || value === null) {
|
|
227
|
+
return undefined;
|
|
228
|
+
}
|
|
229
|
+
if ('hybrid' in value) {
|
|
230
|
+
throw new Error(`[mdorigin] ${configFilePath}: "search.hybrid" has been removed. Use "search.mode" with "hybrid" or "vector" instead.`);
|
|
231
|
+
}
|
|
232
|
+
const searchConfig = value;
|
|
233
|
+
const reranker = typeof searchConfig.reranker === 'object' && searchConfig.reranker !== null
|
|
234
|
+
? normalizeSearchReranker(searchConfig.reranker)
|
|
235
|
+
: undefined;
|
|
236
|
+
const scoreAdjustment = typeof searchConfig.scoreAdjustment === 'object' &&
|
|
237
|
+
searchConfig.scoreAdjustment !== null
|
|
238
|
+
? normalizeSearchScoreAdjustment(searchConfig.scoreAdjustment)
|
|
239
|
+
: undefined;
|
|
240
|
+
const policy = typeof searchConfig.policy === 'object' && searchConfig.policy !== null
|
|
241
|
+
? normalizeSearchPolicy(searchConfig.policy)
|
|
242
|
+
: undefined;
|
|
243
|
+
const normalized = {
|
|
244
|
+
topK: normalizeOptionalPositiveInteger(searchConfig.topK),
|
|
245
|
+
mode: searchConfig.mode === 'hybrid' || searchConfig.mode === 'vector'
|
|
246
|
+
? searchConfig.mode
|
|
247
|
+
: undefined,
|
|
248
|
+
minScore: normalizeOptionalNumber(searchConfig.minScore),
|
|
249
|
+
reranker,
|
|
250
|
+
scoreAdjustment,
|
|
251
|
+
policy,
|
|
252
|
+
};
|
|
253
|
+
return Object.values(normalized).some((entry) => entry !== undefined)
|
|
254
|
+
? normalized
|
|
255
|
+
: undefined;
|
|
256
|
+
}
|
|
257
|
+
function normalizeSearchReranker(value) {
|
|
258
|
+
if (typeof value !== 'object' || value === null) {
|
|
259
|
+
return undefined;
|
|
260
|
+
}
|
|
261
|
+
const reranker = value;
|
|
262
|
+
const normalized = {
|
|
263
|
+
kind: reranker.kind === 'embedding-v1' || reranker.kind === 'heuristic-v1'
|
|
264
|
+
? reranker.kind
|
|
265
|
+
: undefined,
|
|
266
|
+
candidatePoolSize: normalizeOptionalPositiveInteger(reranker.candidatePoolSize),
|
|
267
|
+
};
|
|
268
|
+
return Object.values(normalized).some((entry) => entry !== undefined)
|
|
269
|
+
? normalized
|
|
270
|
+
: undefined;
|
|
271
|
+
}
|
|
272
|
+
function normalizeSearchScoreAdjustment(value) {
|
|
273
|
+
if (typeof value !== 'object' || value === null) {
|
|
274
|
+
return undefined;
|
|
275
|
+
}
|
|
276
|
+
const scoreAdjustment = value;
|
|
277
|
+
const normalized = {
|
|
278
|
+
metadataNumericMultiplier: typeof scoreAdjustment.metadataNumericMultiplier === 'string' &&
|
|
279
|
+
scoreAdjustment.metadataNumericMultiplier !== ''
|
|
280
|
+
? scoreAdjustment.metadataNumericMultiplier
|
|
281
|
+
: undefined,
|
|
282
|
+
};
|
|
283
|
+
return Object.values(normalized).some((entry) => entry !== undefined)
|
|
284
|
+
? normalized
|
|
285
|
+
: undefined;
|
|
286
|
+
}
|
|
287
|
+
function normalizeSearchPolicy(value) {
|
|
288
|
+
if (typeof value !== 'object' || value === null) {
|
|
289
|
+
return undefined;
|
|
290
|
+
}
|
|
291
|
+
const policy = value;
|
|
292
|
+
const normalized = {
|
|
293
|
+
shortQuery: normalizeSearchShortQueryPolicy(policy.shortQuery),
|
|
294
|
+
longQuery: normalizeSearchLongQueryPolicy(policy.longQuery),
|
|
295
|
+
};
|
|
296
|
+
return Object.values(normalized).some((entry) => entry !== undefined)
|
|
297
|
+
? normalized
|
|
298
|
+
: undefined;
|
|
299
|
+
}
|
|
300
|
+
function normalizeSearchShortQueryPolicy(value) {
|
|
301
|
+
if (typeof value !== 'object' || value === null) {
|
|
302
|
+
return undefined;
|
|
303
|
+
}
|
|
304
|
+
const policy = value;
|
|
305
|
+
const maxChars = normalizeOptionalPositiveInteger(policy.maxChars);
|
|
306
|
+
if (maxChars === undefined) {
|
|
307
|
+
return undefined;
|
|
308
|
+
}
|
|
309
|
+
return {
|
|
310
|
+
maxChars,
|
|
311
|
+
...normalizeSearchPolicyOverride(policy),
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
function normalizeSearchLongQueryPolicy(value) {
|
|
315
|
+
if (typeof value !== 'object' || value === null) {
|
|
316
|
+
return undefined;
|
|
317
|
+
}
|
|
318
|
+
const policy = value;
|
|
319
|
+
const minChars = normalizeOptionalPositiveInteger(policy.minChars);
|
|
320
|
+
if (minChars === undefined) {
|
|
321
|
+
return undefined;
|
|
322
|
+
}
|
|
323
|
+
return {
|
|
324
|
+
minChars,
|
|
325
|
+
...normalizeSearchPolicyOverride(policy),
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
function normalizeSearchPolicyOverride(value) {
|
|
329
|
+
const normalized = {};
|
|
330
|
+
if (value.mode === null) {
|
|
331
|
+
normalized.mode = null;
|
|
332
|
+
}
|
|
333
|
+
else if (value.mode === 'hybrid' || value.mode === 'vector') {
|
|
334
|
+
normalized.mode = value.mode;
|
|
335
|
+
}
|
|
336
|
+
if (value.minScore === null) {
|
|
337
|
+
normalized.minScore = null;
|
|
338
|
+
}
|
|
339
|
+
else {
|
|
340
|
+
const minScore = normalizeOptionalNumber(value.minScore);
|
|
341
|
+
if (minScore !== undefined) {
|
|
342
|
+
normalized.minScore = minScore;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
if (value.reranker === null) {
|
|
346
|
+
normalized.reranker = null;
|
|
347
|
+
}
|
|
348
|
+
else if (typeof value.reranker === 'object' && value.reranker !== null) {
|
|
349
|
+
const reranker = normalizeSearchReranker(value.reranker);
|
|
350
|
+
if (reranker !== undefined) {
|
|
351
|
+
normalized.reranker = reranker;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
if (value.scoreAdjustment === null) {
|
|
355
|
+
normalized.scoreAdjustment = null;
|
|
356
|
+
}
|
|
357
|
+
else if (typeof value.scoreAdjustment === 'object' &&
|
|
358
|
+
value.scoreAdjustment !== null) {
|
|
359
|
+
const scoreAdjustment = normalizeSearchScoreAdjustment(value.scoreAdjustment);
|
|
360
|
+
if (scoreAdjustment !== undefined) {
|
|
361
|
+
normalized.scoreAdjustment = scoreAdjustment;
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
return normalized;
|
|
365
|
+
}
|
|
200
366
|
function normalizeLogo(value) {
|
|
201
367
|
if (typeof value !== 'object' ||
|
|
202
368
|
value === null ||
|
package/dist/search.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ResolvedSiteConfig } from './core/site-config.js';
|
|
1
|
+
import type { ResolvedSiteConfig, SiteSearchConfig, SiteSearchRerankerConfig, SiteSearchScoreAdjustmentConfig } from './core/site-config.js';
|
|
2
2
|
type JsonValue = null | boolean | number | string | JsonValue[] | {
|
|
3
3
|
[key: string]: JsonValue;
|
|
4
4
|
};
|
|
@@ -21,8 +21,12 @@ export interface SearchHit {
|
|
|
21
21
|
}
|
|
22
22
|
export interface SearchQueryOptions {
|
|
23
23
|
topK?: number;
|
|
24
|
+
mode?: 'hybrid' | 'vector';
|
|
25
|
+
minScore?: number;
|
|
26
|
+
reranker?: SiteSearchRerankerConfig;
|
|
24
27
|
relativePathPrefix?: string;
|
|
25
28
|
metadata?: Record<string, string>;
|
|
29
|
+
scoreAdjustment?: SiteSearchScoreAdjustmentConfig;
|
|
26
30
|
}
|
|
27
31
|
export interface SearchApi {
|
|
28
32
|
search(query: string, options?: SearchQueryOptions): Promise<SearchHit[]>;
|
|
@@ -71,12 +75,16 @@ export interface SearchBundleOptions {
|
|
|
71
75
|
indexDir: string;
|
|
72
76
|
query: string;
|
|
73
77
|
topK?: number;
|
|
78
|
+
mode?: 'hybrid' | 'vector';
|
|
79
|
+
minScore?: number;
|
|
80
|
+
reranker?: SiteSearchRerankerConfig;
|
|
74
81
|
relativePathPrefix?: string;
|
|
75
82
|
metadata?: Record<string, string>;
|
|
83
|
+
scoreAdjustment?: SiteSearchScoreAdjustmentConfig;
|
|
76
84
|
}
|
|
77
85
|
export declare function buildSearchBundle(options: BuildSearchBundleOptions): Promise<BuildSearchBundleResult>;
|
|
78
86
|
export declare function searchBundle(options: SearchBundleOptions): Promise<SearchHit[]>;
|
|
79
|
-
export declare function createSearchApiFromDirectory(indexDir: string): Promise<SearchApi>;
|
|
80
|
-
export declare function createSearchApiFromBundle(bundleEntries: SearchBundleEntry[]): SearchApi;
|
|
81
|
-
export declare function createSearchApiFromExternalBundle(bundleEntries: ExternalSearchBundleEntry[], loadResponse: (entry: ExternalSearchBundleEntry) => Promise<Response
|
|
87
|
+
export declare function createSearchApiFromDirectory(indexDir: string, defaults?: SiteSearchConfig): Promise<SearchApi>;
|
|
88
|
+
export declare function createSearchApiFromBundle(bundleEntries: SearchBundleEntry[], defaults?: SiteSearchConfig): SearchApi;
|
|
89
|
+
export declare function createSearchApiFromExternalBundle(bundleEntries: ExternalSearchBundleEntry[], loadResponse: (entry: ExternalSearchBundleEntry) => Promise<Response>, defaults?: SiteSearchConfig): SearchApi;
|
|
82
90
|
export {};
|
package/dist/search.js
CHANGED
|
@@ -1,14 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { mkdtemp, mkdir, readdir, readFile, realpath, stat, writeFile, } from 'node:fs/promises';
|
|
3
|
-
import os from 'node:os';
|
|
1
|
+
import { mkdir, readdir, readFile, realpath, stat, writeFile, } from 'node:fs/promises';
|
|
4
2
|
import path from 'node:path';
|
|
5
3
|
import { inferDirectoryContentType } from './core/content-type.js';
|
|
6
4
|
import { getDirectoryIndexCandidates } from './core/directory-index.js';
|
|
7
5
|
import { getDocumentSummary, getDocumentTitle, parseMarkdownDocument, stripMachineOnlyMarkdownComments, stripManagedIndexBlock, } from './core/markdown.js';
|
|
8
6
|
import { isIgnoredContentName } from './core/content-store.js';
|
|
9
7
|
const OVERVIEW_CONTENT_FILENAMES = new Set(['readme.md', 'index.md', 'skill.md']);
|
|
10
|
-
const materializedSearchBundleDirectories = new Set();
|
|
11
|
-
let searchBundleDirectoryCleanupRegistered = false;
|
|
12
8
|
export async function buildSearchBundle(options) {
|
|
13
9
|
const buildModule = await loadIndexbindBuildModule();
|
|
14
10
|
const rootDir = path.resolve(options.rootDir);
|
|
@@ -50,23 +46,24 @@ export async function searchBundle(options) {
|
|
|
50
46
|
const searchContext = await openSearchContextFromDirectory(path.resolve(options.indexDir));
|
|
51
47
|
return rerankSearchHits(hydrateSearchHits(await searchContext.index.search(options.query, buildIndexbindSearchOptions({
|
|
52
48
|
topK: options.topK ?? 10,
|
|
49
|
+
mode: options.mode,
|
|
50
|
+
minScore: options.minScore,
|
|
51
|
+
reranker: options.reranker,
|
|
53
52
|
relativePathPrefix: options.relativePathPrefix,
|
|
54
53
|
metadata: options.metadata,
|
|
54
|
+
scoreAdjustment: options.scoreAdjustment,
|
|
55
55
|
})), searchContext.documentsById));
|
|
56
56
|
}
|
|
57
|
-
export async function createSearchApiFromDirectory(indexDir) {
|
|
57
|
+
export async function createSearchApiFromDirectory(indexDir, defaults) {
|
|
58
58
|
const searchContext = await openSearchContextFromDirectory(path.resolve(indexDir));
|
|
59
59
|
return {
|
|
60
60
|
async search(query, options) {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
relativePathPrefix: options?.relativePathPrefix,
|
|
64
|
-
metadata: options?.metadata,
|
|
65
|
-
})), searchContext.documentsById));
|
|
61
|
+
const searchOptions = resolveSearchQueryOptions(query, defaults, options);
|
|
62
|
+
return rerankSearchHits(hydrateSearchHits(await searchContext.index.search(query, buildIndexbindSearchOptions(searchOptions)), searchContext.documentsById));
|
|
66
63
|
},
|
|
67
64
|
};
|
|
68
65
|
}
|
|
69
|
-
export function createSearchApiFromBundle(bundleEntries) {
|
|
66
|
+
export function createSearchApiFromBundle(bundleEntries, defaults) {
|
|
70
67
|
let searchContextPromise = null;
|
|
71
68
|
return {
|
|
72
69
|
async search(query, options) {
|
|
@@ -74,15 +71,12 @@ export function createSearchApiFromBundle(bundleEntries) {
|
|
|
74
71
|
searchContextPromise = openSearchContextFromBundle(bundleEntries, async (entry) => createInlineSearchBundleResponse(entry));
|
|
75
72
|
}
|
|
76
73
|
const searchContext = await searchContextPromise;
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
relativePathPrefix: options?.relativePathPrefix,
|
|
80
|
-
metadata: options?.metadata,
|
|
81
|
-
})), searchContext.documentsById));
|
|
74
|
+
const searchOptions = resolveSearchQueryOptions(query, defaults, options);
|
|
75
|
+
return rerankSearchHits(hydrateSearchHits(await searchContext.index.search(query, buildIndexbindSearchOptions(searchOptions)), searchContext.documentsById));
|
|
82
76
|
},
|
|
83
77
|
};
|
|
84
78
|
}
|
|
85
|
-
export function createSearchApiFromExternalBundle(bundleEntries, loadResponse) {
|
|
79
|
+
export function createSearchApiFromExternalBundle(bundleEntries, loadResponse, defaults) {
|
|
86
80
|
let searchContextPromise = null;
|
|
87
81
|
return {
|
|
88
82
|
async search(query, options) {
|
|
@@ -90,11 +84,8 @@ export function createSearchApiFromExternalBundle(bundleEntries, loadResponse) {
|
|
|
90
84
|
searchContextPromise = openSearchContextFromBundle(bundleEntries, loadResponse);
|
|
91
85
|
}
|
|
92
86
|
const searchContext = await searchContextPromise;
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
relativePathPrefix: options?.relativePathPrefix,
|
|
96
|
-
metadata: options?.metadata,
|
|
97
|
-
})), searchContext.documentsById));
|
|
87
|
+
const searchOptions = resolveSearchQueryOptions(query, defaults, options);
|
|
88
|
+
return rerankSearchHits(hydrateSearchHits(await searchContext.index.search(query, buildIndexbindSearchOptions(searchOptions)), searchContext.documentsById));
|
|
98
89
|
},
|
|
99
90
|
};
|
|
100
91
|
}
|
|
@@ -364,13 +355,9 @@ async function loadIndexbindCloudflareModule() {
|
|
|
364
355
|
}
|
|
365
356
|
}
|
|
366
357
|
async function openWebIndexFromVirtualBundle(bundleEntries, loadResponse) {
|
|
367
|
-
if (isNodeRuntime()) {
|
|
368
|
-
return openWebIndexFromMaterializedBundle(bundleEntries, loadResponse);
|
|
369
|
-
}
|
|
370
358
|
const baseUrl = 'https://mdorigin-search.invalid/';
|
|
371
|
-
const originalFetch = globalThis.fetch;
|
|
372
359
|
const bundleMap = new Map(bundleEntries.map((entry) => [new URL(entry.path, baseUrl).toString(), entry]));
|
|
373
|
-
|
|
360
|
+
const bundleFetch = async (input, init) => {
|
|
374
361
|
const requestUrl = typeof input === 'string'
|
|
375
362
|
? input
|
|
376
363
|
: input instanceof URL
|
|
@@ -380,23 +367,22 @@ async function openWebIndexFromVirtualBundle(bundleEntries, loadResponse) {
|
|
|
380
367
|
if (entry) {
|
|
381
368
|
return loadResponse(entry);
|
|
382
369
|
}
|
|
383
|
-
return
|
|
370
|
+
return fetch(input, init);
|
|
384
371
|
};
|
|
385
372
|
try {
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
}
|
|
390
|
-
catch (error) {
|
|
391
|
-
if (!isCloudflareWasmImportError(error)) {
|
|
392
|
-
throw error;
|
|
393
|
-
}
|
|
394
|
-
const webModule = await loadIndexbindWebModule();
|
|
395
|
-
return await webModule.openWebIndex(new URL(baseUrl));
|
|
396
|
-
}
|
|
373
|
+
const cloudflareModule = await loadIndexbindCloudflareModule();
|
|
374
|
+
return await cloudflareModule.openWebIndex(new URL(baseUrl), {
|
|
375
|
+
fetch: bundleFetch,
|
|
376
|
+
});
|
|
397
377
|
}
|
|
398
|
-
|
|
399
|
-
|
|
378
|
+
catch (error) {
|
|
379
|
+
if (!isCloudflareWasmImportError(error)) {
|
|
380
|
+
throw error;
|
|
381
|
+
}
|
|
382
|
+
const webModule = await loadIndexbindWebModule();
|
|
383
|
+
return await webModule.openWebIndex(new URL(baseUrl), {
|
|
384
|
+
fetch: bundleFetch,
|
|
385
|
+
});
|
|
400
386
|
}
|
|
401
387
|
}
|
|
402
388
|
async function openSearchContextFromDirectory(indexDir) {
|
|
@@ -414,25 +400,6 @@ async function openSearchContextFromBundle(bundleEntries, loadResponse) {
|
|
|
414
400
|
]);
|
|
415
401
|
return { index, documentsById };
|
|
416
402
|
}
|
|
417
|
-
async function openWebIndexFromMaterializedBundle(bundleEntries, loadResponse) {
|
|
418
|
-
const tempDir = await mkdtemp(path.join(os.tmpdir(), 'mdorigin-search-bundle-'));
|
|
419
|
-
registerMaterializedSearchBundleDirectory(tempDir);
|
|
420
|
-
for (const entry of bundleEntries) {
|
|
421
|
-
const response = await loadResponse(entry);
|
|
422
|
-
if (!response.ok) {
|
|
423
|
-
throw new Error(`Failed to materialize search bundle file ${entry.path}: ${response.status} ${response.statusText}`);
|
|
424
|
-
}
|
|
425
|
-
const outputPath = resolveMaterializedSearchBundlePath(tempDir, entry.path);
|
|
426
|
-
await mkdir(path.dirname(outputPath), { recursive: true });
|
|
427
|
-
if (entry.mediaType.startsWith('text/') || entry.mediaType.includes('json')) {
|
|
428
|
-
await writeFile(outputPath, await response.text(), 'utf8');
|
|
429
|
-
continue;
|
|
430
|
-
}
|
|
431
|
-
await writeFile(outputPath, new Uint8Array(await response.arrayBuffer()));
|
|
432
|
-
}
|
|
433
|
-
const webModule = await loadIndexbindWebModule();
|
|
434
|
-
return webModule.openWebIndex(tempDir);
|
|
435
|
-
}
|
|
436
403
|
function createInlineSearchBundleResponse(entry) {
|
|
437
404
|
const headers = new Headers({ 'content-type': entry.mediaType });
|
|
438
405
|
const binaryBody = decodeBase64(entry.base64 ?? '');
|
|
@@ -457,7 +424,9 @@ function isCloudflareWasmImportError(error) {
|
|
|
457
424
|
visited.add(current);
|
|
458
425
|
if (current instanceof Error) {
|
|
459
426
|
const currentMessage = current.message;
|
|
460
|
-
if (targetMessages.some((message) => currentMessage.includes(message))
|
|
427
|
+
if (targetMessages.some((message) => currentMessage.includes(message)) ||
|
|
428
|
+
(currentMessage.includes('Cannot find module') &&
|
|
429
|
+
currentMessage.includes('indexbind_wasm_bg.js'))) {
|
|
461
430
|
return true;
|
|
462
431
|
}
|
|
463
432
|
}
|
|
@@ -473,6 +442,94 @@ function decodeBase64(value) {
|
|
|
473
442
|
const decoded = atob(value);
|
|
474
443
|
return Uint8Array.from(decoded, (character) => character.charCodeAt(0));
|
|
475
444
|
}
|
|
445
|
+
function resolveSearchQueryOptions(query, defaults, overrides) {
|
|
446
|
+
const policyOverride = resolveSearchPolicyOverride(query, defaults?.policy);
|
|
447
|
+
return mergeSearchQueryOptions(mergePolicySearchOptions(defaults, policyOverride), overrides);
|
|
448
|
+
}
|
|
449
|
+
function resolveSearchPolicyOverride(query, policy) {
|
|
450
|
+
if (!policy) {
|
|
451
|
+
return undefined;
|
|
452
|
+
}
|
|
453
|
+
const queryCharCount = Array.from(query.trim()).length;
|
|
454
|
+
if (queryCharCount === 0) {
|
|
455
|
+
return undefined;
|
|
456
|
+
}
|
|
457
|
+
if (policy.shortQuery &&
|
|
458
|
+
queryCharCount <= policy.shortQuery.maxChars) {
|
|
459
|
+
return policy.shortQuery;
|
|
460
|
+
}
|
|
461
|
+
if (policy.longQuery &&
|
|
462
|
+
queryCharCount >= policy.longQuery.minChars) {
|
|
463
|
+
return policy.longQuery;
|
|
464
|
+
}
|
|
465
|
+
return undefined;
|
|
466
|
+
}
|
|
467
|
+
function mergePolicySearchOptions(defaults, policyOverride) {
|
|
468
|
+
const normalized = {
|
|
469
|
+
topK: defaults?.topK,
|
|
470
|
+
mode: defaults?.mode,
|
|
471
|
+
minScore: defaults?.minScore,
|
|
472
|
+
reranker: defaults?.reranker ? { ...defaults.reranker } : undefined,
|
|
473
|
+
scoreAdjustment: defaults?.scoreAdjustment
|
|
474
|
+
? { ...defaults.scoreAdjustment }
|
|
475
|
+
: undefined,
|
|
476
|
+
};
|
|
477
|
+
if (!policyOverride) {
|
|
478
|
+
return normalized;
|
|
479
|
+
}
|
|
480
|
+
if (policyOverride.mode === null) {
|
|
481
|
+
normalized.mode = undefined;
|
|
482
|
+
}
|
|
483
|
+
else if (policyOverride.mode !== undefined) {
|
|
484
|
+
normalized.mode = policyOverride.mode;
|
|
485
|
+
}
|
|
486
|
+
if (policyOverride.minScore === null) {
|
|
487
|
+
normalized.minScore = undefined;
|
|
488
|
+
}
|
|
489
|
+
else if (policyOverride.minScore !== undefined) {
|
|
490
|
+
normalized.minScore = policyOverride.minScore;
|
|
491
|
+
}
|
|
492
|
+
if (policyOverride.reranker === null) {
|
|
493
|
+
normalized.reranker = undefined;
|
|
494
|
+
}
|
|
495
|
+
else if (policyOverride.reranker) {
|
|
496
|
+
normalized.reranker = {
|
|
497
|
+
...(normalized.reranker ?? {}),
|
|
498
|
+
...policyOverride.reranker,
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
if (policyOverride.scoreAdjustment === null) {
|
|
502
|
+
normalized.scoreAdjustment = undefined;
|
|
503
|
+
}
|
|
504
|
+
else if (policyOverride.scoreAdjustment) {
|
|
505
|
+
normalized.scoreAdjustment = {
|
|
506
|
+
...(normalized.scoreAdjustment ?? {}),
|
|
507
|
+
...policyOverride.scoreAdjustment,
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
return normalized;
|
|
511
|
+
}
|
|
512
|
+
function mergeSearchQueryOptions(defaults, overrides) {
|
|
513
|
+
return {
|
|
514
|
+
topK: overrides?.topK ?? defaults?.topK,
|
|
515
|
+
mode: overrides?.mode ?? defaults?.mode,
|
|
516
|
+
minScore: overrides?.minScore ?? defaults?.minScore,
|
|
517
|
+
reranker: overrides?.reranker || defaults?.reranker
|
|
518
|
+
? {
|
|
519
|
+
...(defaults?.reranker ?? {}),
|
|
520
|
+
...(overrides?.reranker ?? {}),
|
|
521
|
+
}
|
|
522
|
+
: undefined,
|
|
523
|
+
relativePathPrefix: overrides?.relativePathPrefix ?? defaults?.relativePathPrefix,
|
|
524
|
+
metadata: overrides?.metadata ?? defaults?.metadata,
|
|
525
|
+
scoreAdjustment: overrides?.scoreAdjustment || defaults?.scoreAdjustment
|
|
526
|
+
? {
|
|
527
|
+
...(defaults?.scoreAdjustment ?? {}),
|
|
528
|
+
...(overrides?.scoreAdjustment ?? {}),
|
|
529
|
+
}
|
|
530
|
+
: undefined,
|
|
531
|
+
};
|
|
532
|
+
}
|
|
476
533
|
function buildIndexbindSearchOptions(options) {
|
|
477
534
|
const normalized = {};
|
|
478
535
|
if (typeof options.topK === 'number' &&
|
|
@@ -481,12 +538,45 @@ function buildIndexbindSearchOptions(options) {
|
|
|
481
538
|
options.topK > 0) {
|
|
482
539
|
normalized.topK = options.topK;
|
|
483
540
|
}
|
|
541
|
+
if (options.mode === 'hybrid' || options.mode === 'vector') {
|
|
542
|
+
normalized.mode = options.mode;
|
|
543
|
+
}
|
|
544
|
+
if (typeof options.minScore === 'number' && Number.isFinite(options.minScore)) {
|
|
545
|
+
normalized.minScore = options.minScore;
|
|
546
|
+
}
|
|
547
|
+
if (options.reranker) {
|
|
548
|
+
const reranker = {};
|
|
549
|
+
if (options.reranker.kind === 'embedding-v1' ||
|
|
550
|
+
options.reranker.kind === 'heuristic-v1') {
|
|
551
|
+
reranker.kind = options.reranker.kind;
|
|
552
|
+
}
|
|
553
|
+
if (typeof options.reranker.candidatePoolSize === 'number' &&
|
|
554
|
+
Number.isFinite(options.reranker.candidatePoolSize) &&
|
|
555
|
+
Number.isInteger(options.reranker.candidatePoolSize) &&
|
|
556
|
+
options.reranker.candidatePoolSize > 0) {
|
|
557
|
+
reranker.candidatePoolSize = options.reranker.candidatePoolSize;
|
|
558
|
+
}
|
|
559
|
+
if (Object.keys(reranker).length > 0) {
|
|
560
|
+
normalized.reranker = reranker;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
484
563
|
if (typeof options.relativePathPrefix === 'string' && options.relativePathPrefix !== '') {
|
|
485
564
|
normalized.relativePathPrefix = options.relativePathPrefix;
|
|
486
565
|
}
|
|
487
566
|
if (options.metadata && Object.keys(options.metadata).length > 0) {
|
|
488
567
|
normalized.metadata = options.metadata;
|
|
489
568
|
}
|
|
569
|
+
if (options.scoreAdjustment) {
|
|
570
|
+
const scoreAdjustment = {};
|
|
571
|
+
if (typeof options.scoreAdjustment.metadataNumericMultiplier === 'string' &&
|
|
572
|
+
options.scoreAdjustment.metadataNumericMultiplier !== '') {
|
|
573
|
+
scoreAdjustment.metadataNumericMultiplier =
|
|
574
|
+
options.scoreAdjustment.metadataNumericMultiplier;
|
|
575
|
+
}
|
|
576
|
+
if (Object.keys(scoreAdjustment).length > 0) {
|
|
577
|
+
normalized.scoreAdjustment = scoreAdjustment;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
490
580
|
return Object.keys(normalized).length > 0 ? normalized : undefined;
|
|
491
581
|
}
|
|
492
582
|
function hydrateSearchHits(hits, documentsById) {
|
|
@@ -539,48 +629,6 @@ async function loadBundleResponseText(entry, loadResponse) {
|
|
|
539
629
|
}
|
|
540
630
|
return response.text();
|
|
541
631
|
}
|
|
542
|
-
function isNodeRuntime() {
|
|
543
|
-
return (typeof process !== 'undefined' &&
|
|
544
|
-
typeof process.versions === 'object' &&
|
|
545
|
-
typeof process.versions.node === 'string');
|
|
546
|
-
}
|
|
547
|
-
function resolveMaterializedSearchBundlePath(baseDir, entryPath) {
|
|
548
|
-
if (path.isAbsolute(entryPath)) {
|
|
549
|
-
throw new Error(`Search bundle path must be relative: ${entryPath}`);
|
|
550
|
-
}
|
|
551
|
-
const normalizedEntryPath = entryPath.replaceAll('/', path.sep);
|
|
552
|
-
const outputPath = path.resolve(baseDir, normalizedEntryPath);
|
|
553
|
-
const relativeOutputPath = path.relative(baseDir, outputPath);
|
|
554
|
-
if (relativeOutputPath === '' ||
|
|
555
|
-
relativeOutputPath.startsWith(`..${path.sep}`) ||
|
|
556
|
-
relativeOutputPath === '..' ||
|
|
557
|
-
path.isAbsolute(relativeOutputPath)) {
|
|
558
|
-
throw new Error(`Search bundle path escapes materialized bundle directory: ${entryPath}`);
|
|
559
|
-
}
|
|
560
|
-
return outputPath;
|
|
561
|
-
}
|
|
562
|
-
function registerMaterializedSearchBundleDirectory(directoryPath) {
|
|
563
|
-
materializedSearchBundleDirectories.add(directoryPath);
|
|
564
|
-
if (searchBundleDirectoryCleanupRegistered || !isNodeRuntime()) {
|
|
565
|
-
return;
|
|
566
|
-
}
|
|
567
|
-
const cleanup = () => {
|
|
568
|
-
for (const stagedDirectory of materializedSearchBundleDirectories) {
|
|
569
|
-
rmSync(stagedDirectory, { recursive: true, force: true });
|
|
570
|
-
}
|
|
571
|
-
materializedSearchBundleDirectories.clear();
|
|
572
|
-
};
|
|
573
|
-
process.once('exit', cleanup);
|
|
574
|
-
process.once('SIGINT', () => {
|
|
575
|
-
cleanup();
|
|
576
|
-
process.exit(130);
|
|
577
|
-
});
|
|
578
|
-
process.once('SIGTERM', () => {
|
|
579
|
-
cleanup();
|
|
580
|
-
process.exit(143);
|
|
581
|
-
});
|
|
582
|
-
searchBundleDirectoryCleanupRegistered = true;
|
|
583
|
-
}
|
|
584
632
|
function rerankSearchHits(hits) {
|
|
585
633
|
const remaining = [...hits];
|
|
586
634
|
const ordered = [];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mdorigin",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Markdown-first publishing for humans and agents.",
|
|
6
6
|
"repository": {
|
|
@@ -78,6 +78,6 @@
|
|
|
78
78
|
"typescript": "^5.9.2"
|
|
79
79
|
},
|
|
80
80
|
"optionalDependencies": {
|
|
81
|
-
"indexbind": "^0.
|
|
81
|
+
"indexbind": "^0.5.1"
|
|
82
82
|
}
|
|
83
83
|
}
|