@o-a/cms-agent 0.1.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot.d.ts +2 -0
- package/dist/boot.js +3 -1
- package/dist/config.d.ts +0 -1
- package/dist/config.js +0 -1
- package/dist/create-site/cli.js +0 -0
- package/dist/create-site/mint-token-cli.js +0 -0
- package/dist/media/filename.js +4 -1
- package/dist/migrations/index.d.ts +1 -1
- package/dist/migrations/index.js +24 -1
- package/dist/renderer/render-cache.d.ts +10 -0
- package/dist/renderer/render-cache.js +11 -0
- package/dist/renderer/render-page.d.ts +2 -0
- package/dist/renderer/render-page.js +40 -1
- package/dist/routes/admin-redirect.d.ts +5 -0
- package/dist/routes/admin-redirect.js +26 -0
- package/dist/routes/capabilities.js +2 -2
- package/dist/routes/media-public.js +6 -0
- package/dist/routes/preview-revision.js +3 -19
- package/dist/routes/preview.js +0 -18
- package/dist/routes/public.d.ts +2 -0
- package/dist/routes/public.js +25 -30
- package/dist/routes/search-public.d.ts +6 -0
- package/dist/routes/search-public.js +104 -0
- package/dist/routes/search.js +4 -0
- package/dist/routes/sitemap.js +3 -9
- package/dist/schemas/page.schema.json +6 -0
- package/dist/search/drivers/node-sqlite-driver.d.ts +5 -1
- package/dist/search/drivers/node-sqlite-driver.js +2 -2
- package/dist/search/query-content.d.ts +32 -0
- package/dist/search/query-content.js +207 -0
- package/dist/search/rebuild-index.js +248 -55
- package/dist/server-config.d.ts +1 -0
- package/dist/server-config.js +28 -1
- package/dist/server.js +32 -0
- package/dist/services/content-read.js +5 -10
- package/dist/services/delete-content.js +2 -13
- package/dist/services/manage-redirects.js +5 -15
- package/dist/services/migration-runner.js +55 -13
- package/dist/services/publish.js +8 -14
- package/dist/services/rate-limit-config.d.ts +1 -1
- package/dist/services/rate-limit-config.js +6 -4
- package/dist/services/validation.d.ts +0 -1
- package/dist/services/validation.js +0 -11
- package/package.json +2 -2
- package/dist/schemas/post.schema.json +0 -25
|
@@ -9,8 +9,8 @@ export const DRIVER_NAME = 'node:sqlite';
|
|
|
9
9
|
// test/static/static-analysis.test.ts). DatabaseSync's own
|
|
10
10
|
// prepare()/exec()/close() already structurally match SearchDriver,
|
|
11
11
|
// so this is a thin adapter, not a reimplementation.
|
|
12
|
-
export function openNodeSqliteDriver(path) {
|
|
13
|
-
const db = new DatabaseSync(path);
|
|
12
|
+
export function openNodeSqliteDriver(path, options = {}) {
|
|
13
|
+
const db = new DatabaseSync(path, options);
|
|
14
14
|
return {
|
|
15
15
|
exec: (sql) => db.exec(sql),
|
|
16
16
|
prepare: (sql) => db.prepare(sql),
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export type FieldOp = 'eq' | 'gt' | 'gte' | 'lt' | 'lte';
|
|
2
|
+
export interface FieldFilter {
|
|
3
|
+
field: string;
|
|
4
|
+
op: FieldOp;
|
|
5
|
+
value: string;
|
|
6
|
+
}
|
|
7
|
+
export interface SortParam {
|
|
8
|
+
field: string;
|
|
9
|
+
direction: 'asc' | 'desc';
|
|
10
|
+
}
|
|
11
|
+
export interface SearchParams {
|
|
12
|
+
q?: string;
|
|
13
|
+
pageType?: string;
|
|
14
|
+
filters: FieldFilter[];
|
|
15
|
+
sort?: SortParam;
|
|
16
|
+
limit: number;
|
|
17
|
+
offset: number;
|
|
18
|
+
}
|
|
19
|
+
export type FieldValue = string | number | boolean;
|
|
20
|
+
export interface SearchResultItem {
|
|
21
|
+
url: string;
|
|
22
|
+
title: string;
|
|
23
|
+
pageType: string;
|
|
24
|
+
fields: Record<string, FieldValue | FieldValue[]>;
|
|
25
|
+
}
|
|
26
|
+
export interface SearchResponse {
|
|
27
|
+
results: SearchResultItem[];
|
|
28
|
+
limit: number;
|
|
29
|
+
offset: number;
|
|
30
|
+
hasMore: boolean;
|
|
31
|
+
}
|
|
32
|
+
export declare function queryContent(searchIndexPath: string, params: SearchParams): SearchResponse;
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { openNodeSqliteDriver } from "./drivers/node-sqlite-driver.js";
|
|
3
|
+
function toFiniteNumber(raw) {
|
|
4
|
+
const parsed = Number(raw);
|
|
5
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
6
|
+
}
|
|
7
|
+
const COMPARATORS = {
|
|
8
|
+
gt: '>',
|
|
9
|
+
gte: '>=',
|
|
10
|
+
lt: '<',
|
|
11
|
+
lte: '<=',
|
|
12
|
+
};
|
|
13
|
+
// Never passes raw user input straight into FTS5 MATCH - a public
|
|
14
|
+
// search box gets real, messy input (unbalanced quotes, a lone
|
|
15
|
+
// trailing "-", a word FTS5 treats as an operator like "AND" or "NOT")
|
|
16
|
+
// which throws a syntax error against a bare MATCH ? (the previous
|
|
17
|
+
// query-index.ts never guarded this at all). Each whitespace-separated
|
|
18
|
+
// token becomes its own quoted, prefix-matched literal - "hello world"
|
|
19
|
+
// becomes "hello"* AND "world"*, embedded quotes doubled per FTS5's own
|
|
20
|
+
// escaping rule - so the constructed expression can never be
|
|
21
|
+
// misinterpreted as an operator, regardless of what the user typed.
|
|
22
|
+
function buildMatchExpression(q) {
|
|
23
|
+
const tokens = q
|
|
24
|
+
.split(/\s+/)
|
|
25
|
+
.map((token) => token.trim())
|
|
26
|
+
.filter((token) => token.length > 0);
|
|
27
|
+
if (tokens.length === 0) {
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
return tokens.map((token) => `"${token.replace(/"/g, '""')}"*`).join(' AND ');
|
|
31
|
+
}
|
|
32
|
+
// One INNER JOIN per filter - a page must satisfy every filter to
|
|
33
|
+
// appear at all, so a page missing that field entirely (or with a
|
|
34
|
+
// non-matching value) is correctly excluded, not just left with a
|
|
35
|
+
// null comparison. Same typed-column branch logic query-fields.ts
|
|
36
|
+
// (now retired) already proved out for a single filter.
|
|
37
|
+
function buildFilterJoin(alias, filter) {
|
|
38
|
+
const args = [filter.field];
|
|
39
|
+
if (filter.op === 'eq') {
|
|
40
|
+
const branches = [`${alias}.value_text = ?`];
|
|
41
|
+
args.push(filter.value);
|
|
42
|
+
const numeric = toFiniteNumber(filter.value);
|
|
43
|
+
if (numeric !== undefined) {
|
|
44
|
+
branches.push(`${alias}.value_number = ?`);
|
|
45
|
+
args.push(numeric);
|
|
46
|
+
}
|
|
47
|
+
if (filter.value === 'true' || filter.value === 'false') {
|
|
48
|
+
branches.push(`${alias}.value_bool = ?`);
|
|
49
|
+
args.push(filter.value === 'true' ? 1 : 0);
|
|
50
|
+
}
|
|
51
|
+
return {
|
|
52
|
+
sql: `JOIN page_fields ${alias} ON ${alias}.url = f.url AND ${alias}.field_key = ? AND (${branches.join(' OR ')})`,
|
|
53
|
+
args,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
const numeric = toFiniteNumber(filter.value);
|
|
57
|
+
if (numeric === undefined) {
|
|
58
|
+
// The route validates a numeric op against a numeric value before
|
|
59
|
+
// ever calling in - reaching here regardless just means "nothing
|
|
60
|
+
// could possibly match", not an error this layer needs to raise.
|
|
61
|
+
return { sql: `JOIN page_fields ${alias} ON ${alias}.url = f.url AND ${alias}.field_key = ? AND 1 = 0`, args };
|
|
62
|
+
}
|
|
63
|
+
args.push(numeric);
|
|
64
|
+
return {
|
|
65
|
+
sql: `JOIN page_fields ${alias} ON ${alias}.url = f.url AND ${alias}.field_key = ? AND ${alias}.value_number ${COMPARATORS[filter.op]} ?`,
|
|
66
|
+
args,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
// LEFT, not INNER - unlike a filter, sorting by a field a given page
|
|
70
|
+
// doesn't have shouldn't drop that page from the results, just leave
|
|
71
|
+
// it ordered with a null value (SQLite sorts NULL first in ASC order).
|
|
72
|
+
// COALESCE across both typed columns since this layer has no schema in
|
|
73
|
+
// hand at query time to know in advance which one a given field
|
|
74
|
+
// actually uses.
|
|
75
|
+
function buildSortJoin(field) {
|
|
76
|
+
return {
|
|
77
|
+
sql: 'LEFT JOIN page_fields sort_field ON sort_field.url = f.url AND sort_field.field_key = ?',
|
|
78
|
+
args: [field],
|
|
79
|
+
column: 'COALESCE(sort_field.value_number, sort_field.value_text)',
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
function fieldRowValue(row) {
|
|
83
|
+
if (row.value_text !== null) {
|
|
84
|
+
return row.value_text;
|
|
85
|
+
}
|
|
86
|
+
if (row.value_number !== null) {
|
|
87
|
+
return row.value_number;
|
|
88
|
+
}
|
|
89
|
+
if (row.value_bool !== null) {
|
|
90
|
+
return row.value_bool === 1;
|
|
91
|
+
}
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
// The one query surface a front-end talks to directly - covers a
|
|
95
|
+
// blog listing (pageType + sort by publishDate + pagination), a
|
|
96
|
+
// product grid (several ANDed filters + sort by a numeric field), and
|
|
97
|
+
// general site search (q), rather than three narrow endpoints each
|
|
98
|
+
// covering one of those. Two queries total, never N+1: one for the
|
|
99
|
+
// (already paginated) matching urls, one gathering every indexed
|
|
100
|
+
// field for just that page of urls to build each result's own
|
|
101
|
+
// "fields" map - a product grid needs price to render, not just to
|
|
102
|
+
// have matched.
|
|
103
|
+
export function queryContent(searchIndexPath, params) {
|
|
104
|
+
// readOnly: true (below) throws if the file doesn't exist rather
|
|
105
|
+
// than silently auto-creating an empty one the way a normal open
|
|
106
|
+
// would - checked here instead, so "no rebuild has ever run yet" (a
|
|
107
|
+
// real, expected state for a brand new site) reads as an empty
|
|
108
|
+
// result set, not a 500.
|
|
109
|
+
if (!existsSync(searchIndexPath)) {
|
|
110
|
+
return { results: [], limit: params.limit, offset: params.offset, hasMore: false };
|
|
111
|
+
}
|
|
112
|
+
const driver = openNodeSqliteDriver(searchIndexPath, { readOnly: true, timeout: 2000 });
|
|
113
|
+
try {
|
|
114
|
+
const joins = [];
|
|
115
|
+
const joinArgs = [];
|
|
116
|
+
params.filters.forEach((filter, index) => {
|
|
117
|
+
const built = buildFilterJoin(`pf${index}`, filter);
|
|
118
|
+
joins.push(built.sql);
|
|
119
|
+
joinArgs.push(...built.args);
|
|
120
|
+
});
|
|
121
|
+
const where = [];
|
|
122
|
+
const whereArgs = [];
|
|
123
|
+
const matchExpression = params.q ? buildMatchExpression(params.q) : undefined;
|
|
124
|
+
if (matchExpression) {
|
|
125
|
+
// FTS5's own "tbl MATCH expr" special syntax only recognises the
|
|
126
|
+
// real table name here, not an alias (confirmed live - "f MATCH
|
|
127
|
+
// ?" throws "no such column: f" even though ordinary column
|
|
128
|
+
// references through the same alias work fine everywhere else in
|
|
129
|
+
// this query).
|
|
130
|
+
where.push('pages_fts MATCH ?');
|
|
131
|
+
whereArgs.push(matchExpression);
|
|
132
|
+
}
|
|
133
|
+
if (params.pageType !== undefined) {
|
|
134
|
+
where.push('f.page_type = ?');
|
|
135
|
+
whereArgs.push(params.pageType);
|
|
136
|
+
}
|
|
137
|
+
let orderJoin = '';
|
|
138
|
+
const orderJoinArgs = [];
|
|
139
|
+
let orderBy = 'f.url ASC';
|
|
140
|
+
if (params.sort) {
|
|
141
|
+
const built = buildSortJoin(params.sort.field);
|
|
142
|
+
orderJoin = built.sql;
|
|
143
|
+
orderJoinArgs.push(...built.args);
|
|
144
|
+
orderBy = `${built.column} ${params.sort.direction === 'desc' ? 'DESC' : 'ASC'}`;
|
|
145
|
+
}
|
|
146
|
+
else if (matchExpression) {
|
|
147
|
+
// FTS5's own bm25-derived rank: more negative is more relevant,
|
|
148
|
+
// so plain ascending order is "best match first".
|
|
149
|
+
orderBy = 'rank';
|
|
150
|
+
}
|
|
151
|
+
const sql = [
|
|
152
|
+
'SELECT DISTINCT f.url, f.title, f.page_type',
|
|
153
|
+
'FROM pages_fts f',
|
|
154
|
+
...joins,
|
|
155
|
+
orderJoin,
|
|
156
|
+
where.length > 0 ? `WHERE ${where.join(' AND ')}` : '',
|
|
157
|
+
`ORDER BY ${orderBy}`,
|
|
158
|
+
'LIMIT ? OFFSET ?',
|
|
159
|
+
]
|
|
160
|
+
.filter((part) => part !== '')
|
|
161
|
+
.join(' ');
|
|
162
|
+
// Request one extra row to know whether there's a next page,
|
|
163
|
+
// rather than a separate COUNT(*) query - a real, doubled cost on
|
|
164
|
+
// every single paginated request neither stated use case (infinite
|
|
165
|
+
// scroll, a grid's own "next" button) actually needs an exact
|
|
166
|
+
// total for.
|
|
167
|
+
const args = [...joinArgs, ...orderJoinArgs, ...whereArgs, params.limit + 1, params.offset];
|
|
168
|
+
const mainRows = driver.prepare(sql).all(...args);
|
|
169
|
+
const hasMore = mainRows.length > params.limit;
|
|
170
|
+
const pageRows = mainRows.slice(0, params.limit);
|
|
171
|
+
const fieldsByUrl = new Map();
|
|
172
|
+
if (pageRows.length > 0) {
|
|
173
|
+
const placeholders = pageRows.map(() => '?').join(', ');
|
|
174
|
+
const fieldRows = driver
|
|
175
|
+
.prepare(`SELECT url, field_key, value_text, value_number, value_bool FROM page_fields WHERE url IN (${placeholders})`)
|
|
176
|
+
.all(...pageRows.map((row) => row.url));
|
|
177
|
+
for (const row of fieldRows) {
|
|
178
|
+
const value = fieldRowValue(row);
|
|
179
|
+
if (value === undefined) {
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
const entry = fieldsByUrl.get(row.url) ?? {};
|
|
183
|
+
const existing = entry[row.field_key];
|
|
184
|
+
if (existing === undefined) {
|
|
185
|
+
entry[row.field_key] = value;
|
|
186
|
+
}
|
|
187
|
+
else if (Array.isArray(existing)) {
|
|
188
|
+
existing.push(value);
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
entry[row.field_key] = [existing, value];
|
|
192
|
+
}
|
|
193
|
+
fieldsByUrl.set(row.url, entry);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
const results = pageRows.map((row) => ({
|
|
197
|
+
url: row.url,
|
|
198
|
+
title: row.title,
|
|
199
|
+
pageType: row.page_type,
|
|
200
|
+
fields: fieldsByUrl.get(row.url) ?? {},
|
|
201
|
+
}));
|
|
202
|
+
return { results, limit: params.limit, offset: params.offset, hasMore };
|
|
203
|
+
}
|
|
204
|
+
finally {
|
|
205
|
+
driver.close();
|
|
206
|
+
}
|
|
207
|
+
}
|
|
@@ -1,10 +1,20 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { mkdirSync, readFileSync, renameSync, unlinkSync } from 'node:fs';
|
|
2
3
|
import { join } from 'node:path';
|
|
4
|
+
import { setImmediate as yieldToEventLoop } from 'node:timers/promises';
|
|
3
5
|
import { listFilesRecursively } from "../services/fs-walk.js";
|
|
4
|
-
import {
|
|
6
|
+
import { loadThemeSchemas } from "../services/theme-schemas.js";
|
|
5
7
|
import { pagePathToUrl } from "../services/urls.js";
|
|
6
8
|
import { enqueue } from "../services/write-queue.js";
|
|
7
9
|
import { openNodeSqliteDriver } from "./drivers/node-sqlite-driver.js";
|
|
10
|
+
// How many content files the rebuild loop processes between yields to
|
|
11
|
+
// the event loop (see the yield's own comment below for why this
|
|
12
|
+
// exists at all). Large enough that setImmediate's own overhead is
|
|
13
|
+
// negligible next to the real per-file work (a parse plus several
|
|
14
|
+
// SQLite inserts); small enough that no single slice runs long enough
|
|
15
|
+
// to meaningfully stall another request. Not a config knob - no
|
|
16
|
+
// evidence yet this needs to be tunable per site.
|
|
17
|
+
const YIELD_EVERY_N_FILES = 25;
|
|
8
18
|
function collectStrings(value, out) {
|
|
9
19
|
if (typeof value === 'string') {
|
|
10
20
|
out.push(value);
|
|
@@ -39,73 +49,256 @@ function extractBody(instances) {
|
|
|
39
49
|
walk(instances);
|
|
40
50
|
return strings.join(' ');
|
|
41
51
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
52
|
+
// Epoch milliseconds for a date-like string - undefined if it doesn't
|
|
53
|
+
// parse, so a malformed date is silently skipped rather than indexed
|
|
54
|
+
// as a nonsensical NaN row (the same "malformed input skipped, not a
|
|
55
|
+
// rebuild failure" tolerance every other part of this pipeline
|
|
56
|
+
// already has for a bad file or a missing theme type).
|
|
57
|
+
function parseDateValue(value) {
|
|
58
|
+
if (typeof value !== 'string') {
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
const parsed = Date.parse(value);
|
|
62
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
63
|
+
}
|
|
64
|
+
// Turns one raw value into ApiFieldRow entries for a given field key -
|
|
65
|
+
// shared by both theme-flagged fields (extractInstanceApiFields below)
|
|
66
|
+
// and the built-in post envelope fields (extractEnvelopeApiFields),
|
|
67
|
+
// since both need the identical "what does this value actually mean
|
|
68
|
+
// for indexing" logic. Three shapes:
|
|
69
|
+
// - an array: one row per scalar element, all under the same
|
|
70
|
+
// fieldKey (e.g. a post's own "tags") - a plain "eq" filter then
|
|
71
|
+
// matches via the exact same mechanism a single-valued field already
|
|
72
|
+
// uses, no separate array-aware query logic needed anywhere else.
|
|
73
|
+
// - a date-like string (isDateField true - a theme field schema'd
|
|
74
|
+
// "type": "string", "format": "date", reusing the existing format
|
|
75
|
+
// convention, or the post envelope's own publishDate): stored as an
|
|
76
|
+
// epoch-ms number in valueNumber, not text, so range operators work
|
|
77
|
+
// on it through the same numeric path a flagged price field uses.
|
|
78
|
+
// - a plain scalar (string/number/boolean): stored in its own typed
|
|
79
|
+
// column. Anything else (object, null, undefined) has nothing
|
|
80
|
+
// sensible to store or compare and is silently skipped, the same way
|
|
81
|
+
// a malformed schema block already is elsewhere in this pipeline.
|
|
82
|
+
function pushFieldValue(blockType, instanceId, fieldKey, value, isDateField, out) {
|
|
83
|
+
if (Array.isArray(value)) {
|
|
84
|
+
for (const item of value) {
|
|
85
|
+
pushFieldValue(blockType, instanceId, fieldKey, item, isDateField, out);
|
|
86
|
+
}
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
if (typeof value === 'string' && isDateField) {
|
|
90
|
+
const epoch = parseDateValue(value);
|
|
91
|
+
if (epoch !== undefined) {
|
|
92
|
+
out.push({ blockType, instanceId, fieldKey, valueText: null, valueNumber: epoch, valueBool: null });
|
|
93
|
+
}
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
if (typeof value === 'string') {
|
|
97
|
+
out.push({ blockType, instanceId, fieldKey, valueText: value, valueNumber: null, valueBool: null });
|
|
98
|
+
}
|
|
99
|
+
else if (typeof value === 'number') {
|
|
100
|
+
out.push({ blockType, instanceId, fieldKey, valueText: null, valueNumber: value, valueBool: null });
|
|
101
|
+
}
|
|
102
|
+
else if (typeof value === 'boolean') {
|
|
103
|
+
out.push({ blockType, instanceId, fieldKey, valueText: null, valueNumber: null, valueBool: value ? 1 : 0 });
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// Reads schema.properties for the given instance's own type, keeping
|
|
107
|
+
// only properties explicitly flagged "api": true (an unvalidated,
|
|
108
|
+
// theme-authored JSON Schema keyword - same status as "format"/
|
|
109
|
+
// "allowedBlocks", see docs/theme-authoring-guide.md and
|
|
110
|
+
// services/validation.ts's own allowedBlockTypesOf) - and pairs each
|
|
111
|
+
// with its actual value out of instance.settings via pushFieldValue.
|
|
112
|
+
function extractInstanceApiFields(instance, schemaMap, out) {
|
|
113
|
+
const type = typeof instance.type === 'string' ? instance.type : undefined;
|
|
114
|
+
const id = typeof instance.id === 'string' ? instance.id : undefined;
|
|
115
|
+
if (!type || !id) {
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
const properties = schemaMap[type]?.properties;
|
|
119
|
+
if (!properties || typeof properties !== 'object') {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
const settings = (instance.settings && typeof instance.settings === 'object' ? instance.settings : {});
|
|
123
|
+
for (const [key, propSchema] of Object.entries(properties)) {
|
|
124
|
+
const schema = propSchema;
|
|
125
|
+
if (schema?.api !== true) {
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
const isDateField = schema.type === 'string' && schema.format === 'date';
|
|
129
|
+
pushFieldValue(type, id, key, settings[key], isDateField, out);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// Built-in envelope fields, auto-indexed with no "api": true needed -
|
|
133
|
+
// author/publishDate/tags are optional on every page (page.schema.json),
|
|
134
|
+
// so indexing is presence-based rather than gated on a specific "type"
|
|
135
|
+
// value: any page carrying one of these fields gets it indexed,
|
|
136
|
+
// regardless of what its own type string is. block_type '__page__' is a
|
|
137
|
+
// sentinel (never a real theme type, which always matches a *.liquid
|
|
138
|
+
// filename) marking these rows as envelope-level rather than a real
|
|
139
|
+
// section/block instance; instanceId is the page's own url - stable
|
|
140
|
+
// and unique enough, since there's exactly one envelope per page.
|
|
141
|
+
function extractEnvelopeApiFields(page, url) {
|
|
142
|
+
const rows = [];
|
|
143
|
+
pushFieldValue('__page__', url, 'author', page.author, false, rows);
|
|
144
|
+
pushFieldValue('__page__', url, 'publishDate', page.publishDate, true, rows);
|
|
145
|
+
pushFieldValue('__page__', url, 'tags', page.tags, false, rows);
|
|
146
|
+
return rows;
|
|
147
|
+
}
|
|
148
|
+
// Top-level page.sections entries are sections; every level of nested
|
|
149
|
+
// .blocks (arbitrarily deep - instance.schema.json's blocks is self-
|
|
150
|
+
// referential, same reasoning extractBody's own walk above already
|
|
151
|
+
// documents) is a block, so which theme-schema map applies flips
|
|
152
|
+
// exactly once, at the top, and stays fixed for everything nested
|
|
153
|
+
// underneath.
|
|
154
|
+
function extractApiFields(sections, sectionSchemas, blockSchemas) {
|
|
155
|
+
const rows = [];
|
|
156
|
+
const walk = (list, schemaMap) => {
|
|
157
|
+
if (!list) {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
for (const instance of list) {
|
|
161
|
+
extractInstanceApiFields(instance, schemaMap, rows);
|
|
162
|
+
walk(instance.blocks, blockSchemas);
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
walk(sections, sectionSchemas);
|
|
166
|
+
return rows;
|
|
167
|
+
}
|
|
168
|
+
function unlinkIfExists(path) {
|
|
49
169
|
try {
|
|
50
|
-
unlinkSync(
|
|
170
|
+
unlinkSync(path);
|
|
51
171
|
}
|
|
52
172
|
catch (error) {
|
|
53
173
|
if (error.code !== 'ENOENT') {
|
|
54
174
|
throw error;
|
|
55
175
|
}
|
|
56
176
|
}
|
|
57
|
-
|
|
177
|
+
}
|
|
178
|
+
// A fresh sqlite file is never actually left in WAL mode by this
|
|
179
|
+
// module (nothing here turns that on), but cleaning up any stray
|
|
180
|
+
// -wal/-shm sidecar files defensively costs nothing and avoids ever
|
|
181
|
+
// leaving one behind next to an abandoned temp build.
|
|
182
|
+
function cleanupSqliteArtifacts(path) {
|
|
183
|
+
unlinkIfExists(path);
|
|
184
|
+
unlinkIfExists(`${path}-wal`);
|
|
185
|
+
unlinkIfExists(`${path}-shm`);
|
|
186
|
+
}
|
|
187
|
+
async function rebuildIndexJob(config) {
|
|
188
|
+
mkdirSync(config.dataRoot, { recursive: true });
|
|
189
|
+
// Built into a fresh temp file, then renamed atomically over the
|
|
190
|
+
// real path (below) - not the old delete-then-recreate-in-place
|
|
191
|
+
// approach, which left a real window where a concurrent read saw
|
|
192
|
+
// either a missing file or one that exists but has no tables in it
|
|
193
|
+
// yet. POSIX rename() is atomic: a reader always sees either the
|
|
194
|
+
// complete old index or the complete new one, never an in-between
|
|
195
|
+
// state, and never has to retry an open that landed in the gap.
|
|
196
|
+
const tmpPath = `${config.searchIndexPath}.tmp-${randomUUID()}`;
|
|
197
|
+
cleanupSqliteArtifacts(tmpPath);
|
|
198
|
+
const themeSchemas = loadThemeSchemas(config.themeRoot);
|
|
58
199
|
try {
|
|
59
|
-
driver
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
200
|
+
const driver = openNodeSqliteDriver(tmpPath);
|
|
201
|
+
try {
|
|
202
|
+
driver.exec('CREATE VIRTUAL TABLE pages_fts USING fts5(url UNINDEXED, title, body, page_type UNINDEXED)');
|
|
203
|
+
// A plain table, not FTS5 - page_fields holds typed, exact/range-
|
|
204
|
+
// comparable values (a price, a rating), the opposite of pages_fts's
|
|
205
|
+
// own free-text matching. One row per exposed field per instance
|
|
206
|
+
// (not one column per field name): a page can carry several
|
|
207
|
+
// instances of the same block type (several "product" blocks on one
|
|
208
|
+
// listing page), each with its own value, and different pages may
|
|
209
|
+
// expose entirely different field sets - a fixed column-per-field
|
|
210
|
+
// schema can't accommodate either. No foreign key back to
|
|
211
|
+
// pages_fts.url - this whole index is disposable and rebuilt wholly
|
|
212
|
+
// from scratch every time, same as pages_fts itself.
|
|
213
|
+
driver.exec('CREATE TABLE page_fields (url TEXT NOT NULL, block_type TEXT NOT NULL, instance_id TEXT NOT NULL, field_key TEXT NOT NULL, value_text TEXT, value_number REAL, value_bool INTEGER)');
|
|
214
|
+
// Composite, not a bare field_key index - field_key alone only
|
|
215
|
+
// narrows to one field's rows; leading with it here still serves
|
|
216
|
+
// that same narrowing (the leftmost-column rule), but the second
|
|
217
|
+
// column also covers the typed value comparison itself (an
|
|
218
|
+
// equality or range check) without a further per-row scan. url
|
|
219
|
+
// supports the self-join a multi-field query ANDs together
|
|
220
|
+
// (queryContent, query-content.ts) - with no index there, ANDing a
|
|
221
|
+
// second filter means scanning page_fields in full for every row
|
|
222
|
+
// the first filter matched.
|
|
223
|
+
driver.exec('CREATE INDEX page_fields_key_number ON page_fields (field_key, value_number)');
|
|
224
|
+
driver.exec('CREATE INDEX page_fields_key_text ON page_fields (field_key, value_text)');
|
|
225
|
+
driver.exec('CREATE INDEX page_fields_url ON page_fields (url)');
|
|
226
|
+
const insert = driver.prepare('INSERT INTO pages_fts (url, title, body, page_type) VALUES (?, ?, ?, ?)');
|
|
227
|
+
const insertField = driver.prepare('INSERT INTO page_fields (url, block_type, instance_id, field_key, value_text, value_number, value_bool) VALUES (?, ?, ?, ?, ?, ?, ?)');
|
|
228
|
+
// Menus are deliberately never walked here at all: they have no
|
|
229
|
+
// public URL to point a search result at.
|
|
230
|
+
const collections = [{ root: config.pagesRoot, toUrl: pagePathToUrl }];
|
|
231
|
+
driver.exec('BEGIN');
|
|
232
|
+
let filesExamined = 0;
|
|
233
|
+
for (const { root, toUrl } of collections) {
|
|
234
|
+
for (const relativePath of listFilesRecursively(root, root, '.json')) {
|
|
235
|
+
// A genuine macrotask yield (setImmediate, not a microtask like
|
|
236
|
+
// Promise.resolve()/queueMicrotask - Node drains every queued
|
|
237
|
+
// microtask before the event loop ever reaches its I/O phases,
|
|
238
|
+
// so a chain of only-microtask yields still fully blocks an
|
|
239
|
+
// incoming HTTP request from being processed). Without this,
|
|
240
|
+
// this loop's entire body - potentially thousands of files -
|
|
241
|
+
// runs as one uninterruptible synchronous block: since Node is
|
|
242
|
+
// single-threaded, that means every other request the server
|
|
243
|
+
// is handling (auth, content reads, publishes) stalls for the
|
|
244
|
+
// rebuild's whole duration, not just other search queries.
|
|
245
|
+
// Counted once per file examined regardless of whether it
|
|
246
|
+
// ends up skipped below, so the cadence tracks total work
|
|
247
|
+
// done, not just files actually indexed.
|
|
248
|
+
filesExamined += 1;
|
|
249
|
+
if (filesExamined % YIELD_EVERY_N_FILES === 0) {
|
|
250
|
+
await yieldToEventLoop();
|
|
251
|
+
}
|
|
252
|
+
let page;
|
|
253
|
+
try {
|
|
254
|
+
page = JSON.parse(readFileSync(join(root, relativePath), 'utf-8'));
|
|
255
|
+
}
|
|
256
|
+
catch {
|
|
257
|
+
// A malformed individual file is skipped, not an all-or-nothing
|
|
258
|
+
// abort: the index is explicitly disposable/best-effort, and
|
|
259
|
+
// aborting the whole rebuild over one bad file would leave no
|
|
260
|
+
// working index at all - strictly worse than skipping one page.
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
// Never walks draftsRoot at all, and skips unpublished content
|
|
264
|
+
// here - both halves of "drafts and unpublished content are
|
|
265
|
+
// absent from the index" (G3) are true by construction, not by
|
|
266
|
+
// a filter that could be gotten wrong.
|
|
267
|
+
if (page.published === false) {
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
const url = toUrl(relativePath);
|
|
271
|
+
const title = typeof page.title === 'string' ? page.title : '';
|
|
272
|
+
const pageType = typeof page.type === 'string' ? page.type : '';
|
|
273
|
+
const body = extractBody(page.sections);
|
|
274
|
+
insert.run(url, title, body, pageType);
|
|
275
|
+
const apiFields = [
|
|
276
|
+
...extractApiFields(page.sections, themeSchemas.sections, themeSchemas.blocks),
|
|
277
|
+
...extractEnvelopeApiFields(page, url),
|
|
278
|
+
];
|
|
279
|
+
for (const row of apiFields) {
|
|
280
|
+
insertField.run(url, row.blockType, row.instanceId, row.fieldKey, row.valueText, row.valueNumber, row.valueBool);
|
|
281
|
+
}
|
|
75
282
|
}
|
|
76
|
-
catch {
|
|
77
|
-
// A malformed individual file is skipped, not an all-or-nothing
|
|
78
|
-
// abort: the index is explicitly disposable/best-effort, and
|
|
79
|
-
// aborting the whole rebuild over one bad file would leave no
|
|
80
|
-
// working index at all - strictly worse than skipping one page.
|
|
81
|
-
continue;
|
|
82
|
-
}
|
|
83
|
-
// Never walks draftsRoot at all, and skips unpublished content
|
|
84
|
-
// here - both halves of "drafts and unpublished content are
|
|
85
|
-
// absent from the index" (G3) are true by construction, not by
|
|
86
|
-
// a filter that could be gotten wrong.
|
|
87
|
-
if (page.published === false) {
|
|
88
|
-
continue;
|
|
89
|
-
}
|
|
90
|
-
const url = toUrl(relativePath);
|
|
91
|
-
const title = typeof page.title === 'string' ? page.title : '';
|
|
92
|
-
const body = extractBody(page.sections);
|
|
93
|
-
insert.run(url, title, body);
|
|
94
283
|
}
|
|
284
|
+
driver.exec('COMMIT');
|
|
95
285
|
}
|
|
96
|
-
|
|
286
|
+
finally {
|
|
287
|
+
driver.close();
|
|
288
|
+
}
|
|
289
|
+
renameSync(tmpPath, config.searchIndexPath);
|
|
97
290
|
}
|
|
98
|
-
|
|
99
|
-
|
|
291
|
+
catch (error) {
|
|
292
|
+
cleanupSqliteArtifacts(tmpPath);
|
|
293
|
+
throw error;
|
|
100
294
|
}
|
|
101
295
|
}
|
|
102
|
-
// Queued via enqueue()
|
|
103
|
-
//
|
|
104
|
-
//
|
|
105
|
-
//
|
|
106
|
-
//
|
|
107
|
-
//
|
|
108
|
-
// reusing it for self-exclusion costs nothing.
|
|
296
|
+
// Queued via enqueue() - two concurrent rebuilds could otherwise both
|
|
297
|
+
// build their own temp file and both attempt the final rename; the
|
|
298
|
+
// second rename would still win cleanly (rename() just replaces
|
|
299
|
+
// whatever is there), but the first rebuild's now-orphaned temp file
|
|
300
|
+
// would never get cleaned up. enqueue() is a generic, domain-agnostic
|
|
301
|
+
// primitive, so reusing it for self-exclusion costs nothing.
|
|
109
302
|
export function rebuildIndex(config) {
|
|
110
303
|
return enqueue(() => rebuildIndexJob(config));
|
|
111
304
|
}
|
package/dist/server-config.d.ts
CHANGED
package/dist/server-config.js
CHANGED
|
@@ -128,6 +128,31 @@ function parseIpAllowlist(value) {
|
|
|
128
128
|
});
|
|
129
129
|
return value;
|
|
130
130
|
}
|
|
131
|
+
// Undefined -> undefined: absence means the /admin redirect feature
|
|
132
|
+
// is off entirely, not a default target to redirect to (there's no
|
|
133
|
+
// sensible default admin URL to assume). Validated as a real absolute
|
|
134
|
+
// http(s) URL, not just any non-empty string, since it becomes a
|
|
135
|
+
// redirect target - rejects `javascript:`/`ftp:`/anything else before
|
|
136
|
+
// it can ever reach a Location header.
|
|
137
|
+
function parseAdminBaseUrl(value) {
|
|
138
|
+
if (value === undefined) {
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
142
|
+
throw new StartupCheckError('invalid-site-config', `site.config.json's "adminBaseUrl" must be a non-empty string, got ${JSON.stringify(value)}`);
|
|
143
|
+
}
|
|
144
|
+
let parsed;
|
|
145
|
+
try {
|
|
146
|
+
parsed = new URL(value);
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
throw new StartupCheckError('invalid-site-config', `site.config.json's "adminBaseUrl" must be a valid URL, got ${JSON.stringify(value)}`);
|
|
150
|
+
}
|
|
151
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
152
|
+
throw new StartupCheckError('invalid-site-config', `site.config.json's "adminBaseUrl" must be an http or https URL, got ${JSON.stringify(value)}`);
|
|
153
|
+
}
|
|
154
|
+
return value;
|
|
155
|
+
}
|
|
131
156
|
function parseCheckpointIntervalMs(value) {
|
|
132
157
|
if (value === undefined) {
|
|
133
158
|
return DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
@@ -182,6 +207,7 @@ export function loadServerConfig(siteRoot) {
|
|
|
182
207
|
ipAllowlist: [],
|
|
183
208
|
checkpointIntervalMs: DEFAULT_CHECKPOINT_INTERVAL_MS,
|
|
184
209
|
media: { maxUploadBytes: DEFAULT_MEDIA_MAX_UPLOAD_BYTES },
|
|
210
|
+
adminBaseUrl: undefined,
|
|
185
211
|
};
|
|
186
212
|
}
|
|
187
213
|
let parsed;
|
|
@@ -206,5 +232,6 @@ export function loadServerConfig(siteRoot) {
|
|
|
206
232
|
const ipAllowlist = parseIpAllowlist(record.ipAllowlist);
|
|
207
233
|
const checkpointIntervalMs = parseCheckpointIntervalMs(record.checkpointIntervalMs);
|
|
208
234
|
const media = parseMedia(record.media);
|
|
209
|
-
|
|
235
|
+
const adminBaseUrl = parseAdminBaseUrl(record.adminBaseUrl);
|
|
236
|
+
return { port: resolvePort(port), tokens, rateLimit, trustProxy, ipAllowlist, checkpointIntervalMs, media, adminBaseUrl };
|
|
210
237
|
}
|