@caelo-cms/shared 0.12.9 → 0.12.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-cost.d.ts +146 -0
- package/dist/ai-cost.d.ts.map +1 -0
- package/dist/ai-cost.js +182 -0
- package/dist/ai-cost.js.map +1 -0
- package/dist/ai-tools.d.ts +1 -1
- package/dist/build-page.d.ts +2 -2
- package/dist/cap-failures.d.ts.map +1 -1
- package/dist/cap-failures.js +7 -5
- package/dist/cap-failures.js.map +1 -1
- package/dist/change-journal.d.ts.map +1 -1
- package/dist/change-journal.js +16 -0
- package/dist/change-journal.js.map +1 -1
- package/dist/content.d.ts +3 -3
- package/dist/embedding-models.d.ts +95 -0
- package/dist/embedding-models.d.ts.map +1 -0
- package/dist/embedding-models.js +60 -0
- package/dist/embedding-models.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/module-id-tag.d.ts +13 -0
- package/dist/module-id-tag.d.ts.map +1 -1
- package/dist/module-id-tag.js +24 -2
- package/dist/module-id-tag.js.map +1 -1
- package/dist/preview-compose.d.ts +17 -1
- package/dist/preview-compose.d.ts.map +1 -1
- package/dist/preview-compose.js +10 -3
- package/dist/preview-compose.js.map +1 -1
- package/dist/search-feed.d.ts +123 -0
- package/dist/search-feed.d.ts.map +1 -0
- package/dist/search-feed.js +152 -0
- package/dist/search-feed.js.map +1 -0
- package/dist/seo.d.ts +19 -0
- package/dist/seo.d.ts.map +1 -1
- package/dist/seo.js +12 -1
- package/dist/seo.js.map +1 -1
- package/dist/site-security-headers.d.ts +2 -2
- package/dist/stage-status.d.ts +10 -5
- package/dist/stage-status.d.ts.map +1 -1
- package/dist/stage-status.js +27 -1
- package/dist/stage-status.js.map +1 -1
- package/dist/static-files.d.ts.map +1 -1
- package/dist/static-files.js +1 -0
- package/dist/static-files.js.map +1 -1
- package/dist/version.d.ts +2 -2
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/dist/visitor-ai-origin.d.ts +41 -0
- package/dist/visitor-ai-origin.d.ts.map +1 -0
- package/dist/visitor-ai-origin.js +65 -0
- package/dist/visitor-ai-origin.js.map +1 -0
- package/package.json +8 -1
- package/src/ai-cost.test.ts +81 -0
- package/src/ai-cost.ts +288 -0
- package/src/cap-failures.ts +7 -5
- package/src/change-journal.ts +31 -0
- package/src/embedding-models.test.ts +37 -0
- package/src/embedding-models.ts +147 -0
- package/src/index.ts +9 -0
- package/src/module-id-tag.test.ts +27 -1
- package/src/module-id-tag.ts +29 -2
- package/src/preview-compose.test.ts +21 -0
- package/src/preview-compose.ts +33 -4
- package/src/search-feed.test.ts +115 -0
- package/src/search-feed.ts +180 -0
- package/src/seo.test.ts +13 -0
- package/src/seo.ts +26 -1
- package/src/stage-status.ts +29 -1
- package/src/static-files.ts +1 -0
- package/src/version.ts +1 -1
- package/src/visitor-ai-origin.ts +86 -0
package/src/change-journal.ts
CHANGED
|
@@ -277,6 +277,8 @@ export const JOURNAL_EXEMPT_TABLES: Readonly<Record<string, string>> = {
|
|
|
277
277
|
email_config: "Owner email configuration; read per send",
|
|
278
278
|
mail_delivery_caps:
|
|
279
279
|
"#1337: the Owner's delivery caps of the public mail outboxes; read per drain",
|
|
280
|
+
visitor_ai_settings:
|
|
281
|
+
"#1654: the Owner's visitor-AI settings; read per call, and published to the gateways by its projection trigger",
|
|
280
282
|
public_mail_deliveries:
|
|
281
283
|
"#1337: the ledger of mails the admin sent from the public mail outboxes; only counted for the delivery caps",
|
|
282
284
|
telemetry_settings: "the install's telemetry consent",
|
|
@@ -286,12 +288,40 @@ export const JOURNAL_EXEMPT_TABLES: Readonly<Record<string, string>> = {
|
|
|
286
288
|
release_check_cache: RUNTIME,
|
|
287
289
|
deploy_targets: "deploy target configuration; read by the deploy ops",
|
|
288
290
|
deploy_runs: "deploy runs; reach editors as the op-classified `deploys` topic",
|
|
291
|
+
live_page_versions:
|
|
292
|
+
"the record of what production serves per page, appended only by the runs that change production (docs/dev/search-engine-feed.md); nothing is derived from its changes",
|
|
293
|
+
staged_page_changes:
|
|
294
|
+
"what a Stage found changed against live_page_versions, read once by the Publish live that ships it",
|
|
295
|
+
search_engine_notifications:
|
|
296
|
+
"the WebSub and IndexNow sends of production runs; read by their sender and Settings → SEO",
|
|
289
297
|
redirect_sync_state: RUNTIME,
|
|
290
298
|
projection_boot_publish: RUNTIME,
|
|
291
299
|
gateway_refresh_state: RUNTIME,
|
|
292
300
|
redirect_changes:
|
|
293
301
|
"which transactions changed redirects, folded by the redirect sync; the journaled redirects table is the source",
|
|
294
302
|
static_bakes: "derived build output of plugin-baked regions",
|
|
303
|
+
collection_changes:
|
|
304
|
+
"plugin collection writes waiting for an index (#1648): the indexing job consumes them, nothing reads them as state",
|
|
305
|
+
stage_output_commits:
|
|
306
|
+
"build state of a Stage (#1647): which Stages' page output counts, in build order",
|
|
307
|
+
collection_release_changes:
|
|
308
|
+
"build state of a Stage (#1648): which changes each collection release is built from",
|
|
309
|
+
collection_releases:
|
|
310
|
+
"build state of a Stage like deploy_runs (#1648): which collection release each run produced; the projection publishes the pointed run's",
|
|
311
|
+
plugin_page_observations:
|
|
312
|
+
"derived from Stage output (#1647): the hashes an observer last received; losing it only re-sends pages at the next Stage",
|
|
313
|
+
plugin_page_observation_pending:
|
|
314
|
+
"a running Stage's observer deliveries (#1647), committed or dropped with the run",
|
|
315
|
+
plugin_module_counts:
|
|
316
|
+
"derived from Stage output (#1647): the placement counts an observer last received",
|
|
317
|
+
plugin_module_count_pending:
|
|
318
|
+
"a running Stage's placement counts (#1647), committed or dropped with the run",
|
|
319
|
+
plugin_page_observers:
|
|
320
|
+
"derived from Stage output (#1647): the digest an observer last saw every page under; losing it only re-sends every page at the next Stage",
|
|
321
|
+
plugin_page_observer_pending:
|
|
322
|
+
"a running Stage's observer digests (#1647), committed or dropped with the run",
|
|
323
|
+
plugin_batch_runs:
|
|
324
|
+
"the run log of plugin batch operations (#1646): queued, running and how each ended; read by its list op, nothing is derived from it",
|
|
295
325
|
content_instance_refs:
|
|
296
326
|
"derived (synchronous, #1287): the ids and media slugs each content instance's values reference, maintained by triggers on the journaled content_instances; rebuilt by content_instances.rebuild_refs",
|
|
297
327
|
content_instance_snapshot_refs:
|
|
@@ -354,6 +384,7 @@ export const JOURNAL_EXEMPT_TABLES: Readonly<Record<string, string>> = {
|
|
|
354
384
|
deploy_pending_actions: QUEUE,
|
|
355
385
|
domain_pending_actions: QUEUE,
|
|
356
386
|
static_file_pending_actions: QUEUE,
|
|
387
|
+
plugin_config_pending_actions: QUEUE,
|
|
357
388
|
redirect_pending_actions: QUEUE,
|
|
358
389
|
email_config_pending_actions: QUEUE,
|
|
359
390
|
layout_pending_actions: QUEUE,
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
/** #1650 — the embedding-model catalog holds only what may ship. */
|
|
4
|
+
|
|
5
|
+
import { describe, expect, it } from "bun:test";
|
|
6
|
+
import {
|
|
7
|
+
DEFAULT_EMBEDDING_MODEL_ID,
|
|
8
|
+
EMBEDDING_MODEL_IDS,
|
|
9
|
+
EMBEDDING_MODELS,
|
|
10
|
+
embeddingModel,
|
|
11
|
+
embeddingModelFingerprint,
|
|
12
|
+
} from "./embedding-models.js";
|
|
13
|
+
|
|
14
|
+
describe("embedding-model catalog", () => {
|
|
15
|
+
it("has unique ids and a default from the catalog", () => {
|
|
16
|
+
expect(new Set(EMBEDDING_MODEL_IDS).size).toBe(EMBEDDING_MODELS.length);
|
|
17
|
+
expect(embeddingModel(DEFAULT_EMBEDDING_MODEL_ID)).toBeDefined();
|
|
18
|
+
expect(embeddingModel("jina-embeddings-v3")).toBeUndefined();
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
for (const m of EMBEDDING_MODELS) {
|
|
22
|
+
it(`${m.id}: licence, pins and shape`, () => {
|
|
23
|
+
// CLAUDE.md §3: MIT or Apache-2.0 only.
|
|
24
|
+
expect(["MIT", "Apache-2.0"]).toContain(m.licence);
|
|
25
|
+
expect(m.id).toMatch(/^[a-z0-9][a-z0-9.-]{0,79}$/);
|
|
26
|
+
expect(m.source.revision).toMatch(/^[0-9a-f]{40}$/);
|
|
27
|
+
for (const f of [m.files.model, m.files.tokenizer]) {
|
|
28
|
+
expect(f.sha256).toMatch(/^[0-9a-f]{64}$/);
|
|
29
|
+
expect(f.bytes).toBeGreaterThan(0);
|
|
30
|
+
expect(f.name).toMatch(/^[a-z_]+\.(onnx|json)$/);
|
|
31
|
+
}
|
|
32
|
+
expect(m.dims % 32).toBe(0);
|
|
33
|
+
expect(m.maxTokens).toBeGreaterThan(16);
|
|
34
|
+
expect(embeddingModelFingerprint(m)).toBe(`${m.id}@${m.files.model.sha256.slice(0, 12)}`);
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
});
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The core catalog of embedding models (epic #1644, R7, issue #1650).
|
|
5
|
+
*
|
|
6
|
+
* A plugin that wants semantic search names a model of this catalog in its
|
|
7
|
+
* configuration (a `catalogModel` config leaf); no plugin ships a model or
|
|
8
|
+
* a binary. Each entry pins the publisher's repository at one revision and
|
|
9
|
+
* every file by SHA-256, so what runs is what was reviewed: the files are
|
|
10
|
+
* fetched once, at image build (`packages/models/scripts/fetch-models.ts`),
|
|
11
|
+
* verified, and verified again when a process loads them. Only MIT and
|
|
12
|
+
* Apache-2.0 models qualify (CLAUDE.md §3); the licence and its notice ship
|
|
13
|
+
* with the files (THIRD_PARTY_NOTICES).
|
|
14
|
+
*
|
|
15
|
+
* This module is data only (no runtime dependency): the plugin SDK
|
|
16
|
+
* validates config values against {@link EMBEDDING_MODEL_IDS}, the admin
|
|
17
|
+
* shows the entries, and `@caelo-cms/models` loads them (natively in the
|
|
18
|
+
* index job, as WebAssembly in the gateway).
|
|
19
|
+
*
|
|
20
|
+
* Why one model: every entry ships in the admin and the gateway image
|
|
21
|
+
* (about 125 MB each), so an entry is added when a measurement shows a
|
|
22
|
+
* site needs what the default cannot do, not to offer a choice for its
|
|
23
|
+
* own sake. `packages/collection-index/bench/semantic.ts` measures the
|
|
24
|
+
* candidates on a labelled German/English site-search set;
|
|
25
|
+
* `bench/semantic-results.md` holds the numbers that chose the default.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
/** One file of a model, pinned by content. */
|
|
29
|
+
export interface EmbeddingModelFile {
|
|
30
|
+
/** File name in the model's directory (`model.onnx`). */
|
|
31
|
+
readonly name: string;
|
|
32
|
+
/** Path in the source repository at {@link EmbeddingModel.source}'s revision. */
|
|
33
|
+
readonly path: string;
|
|
34
|
+
/** SHA-256 of the file, lowercase hex. */
|
|
35
|
+
readonly sha256: string;
|
|
36
|
+
readonly bytes: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** One approved embedding model. */
|
|
40
|
+
export interface EmbeddingModel {
|
|
41
|
+
/** Stable id plugins name in their configuration. */
|
|
42
|
+
readonly id: string;
|
|
43
|
+
/** What the operator sees ("Granite Embedding 97M, multilingual"). */
|
|
44
|
+
readonly label: string;
|
|
45
|
+
/** One sentence for the operator and the AI: what it is good at. */
|
|
46
|
+
readonly description: string;
|
|
47
|
+
readonly source: {
|
|
48
|
+
readonly host: "huggingface.co";
|
|
49
|
+
readonly repo: string;
|
|
50
|
+
/** The repository commit the files are taken from. */
|
|
51
|
+
readonly revision: string;
|
|
52
|
+
};
|
|
53
|
+
readonly licence: "MIT" | "Apache-2.0";
|
|
54
|
+
readonly licenceUrl: string;
|
|
55
|
+
/** The copyright line the notice names. */
|
|
56
|
+
readonly copyright: string;
|
|
57
|
+
readonly files: {
|
|
58
|
+
/** The ONNX graph; loaded from bytes, so it can name no external file. */
|
|
59
|
+
readonly model: EmbeddingModelFile;
|
|
60
|
+
/** The tokenizer (`tokenizer.json`, Hugging Face tokenizers format). */
|
|
61
|
+
readonly tokenizer: EmbeddingModelFile;
|
|
62
|
+
};
|
|
63
|
+
/** Vector dimensions. */
|
|
64
|
+
readonly dims: number;
|
|
65
|
+
/** Tokens embedded per text, special tokens included; a longer text is cut. */
|
|
66
|
+
readonly maxTokens: number;
|
|
67
|
+
/** How token states become the text's vector (then L2-normalised). */
|
|
68
|
+
readonly pooling: "cls" | "mean";
|
|
69
|
+
/** Prepended to the text (e5 models were trained with `query: ` / `passage: `). */
|
|
70
|
+
readonly prefixes: { readonly query: string; readonly passage: string };
|
|
71
|
+
/** The tokenizer's padding token id. */
|
|
72
|
+
readonly padTokenId: number;
|
|
73
|
+
/** The graph's inputs, in the names the export uses. */
|
|
74
|
+
readonly inputs: readonly ("input_ids" | "attention_mask" | "token_type_ids")[];
|
|
75
|
+
/** The graph output holding per-token states (`[batch, tokens, dims]`). */
|
|
76
|
+
readonly output: string;
|
|
77
|
+
/** Languages it serves well, as the publisher documents them. */
|
|
78
|
+
readonly languages: string;
|
|
79
|
+
/** How the published file is quantised. */
|
|
80
|
+
readonly quantization: string;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Granite Embedding 97M multilingual R2 (IBM, Apache-2.0): chosen by the
|
|
85
|
+
* #1650 measurements (bench/semantic-results.md).
|
|
86
|
+
*/
|
|
87
|
+
const GRANITE_97M_R2: EmbeddingModel = {
|
|
88
|
+
id: "granite-embedding-97m-multilingual-r2",
|
|
89
|
+
label: "Granite Embedding 97M, multilingual",
|
|
90
|
+
description:
|
|
91
|
+
"Small multilingual model (52 languages with retrieval training, German and English among them); finds pages by meaning and across languages.",
|
|
92
|
+
source: {
|
|
93
|
+
host: "huggingface.co",
|
|
94
|
+
repo: "ibm-granite/granite-embedding-97m-multilingual-r2",
|
|
95
|
+
revision: "835ad14087e140460703cf0fae09f97d469d65c2",
|
|
96
|
+
},
|
|
97
|
+
licence: "Apache-2.0",
|
|
98
|
+
licenceUrl: "https://www.apache.org/licenses/LICENSE-2.0",
|
|
99
|
+
copyright: "Copyright IBM Corp.",
|
|
100
|
+
files: {
|
|
101
|
+
model: {
|
|
102
|
+
name: "model.onnx",
|
|
103
|
+
path: "onnx/model_quint8_avx2.onnx",
|
|
104
|
+
sha256: "a6022dd8220ea6f6595562a1328ee216f4a94faa55362f2f4747c80f1e78772e",
|
|
105
|
+
bytes: 98_247_878,
|
|
106
|
+
},
|
|
107
|
+
tokenizer: {
|
|
108
|
+
name: "tokenizer.json",
|
|
109
|
+
path: "tokenizer.json",
|
|
110
|
+
sha256: "4f2842d568e2724370aec203652a42ac783c7937f8347a1a2cc7506d71f1582f",
|
|
111
|
+
bytes: 25_301_672,
|
|
112
|
+
},
|
|
113
|
+
},
|
|
114
|
+
dims: 384,
|
|
115
|
+
maxTokens: 512,
|
|
116
|
+
pooling: "cls",
|
|
117
|
+
prefixes: { query: "", passage: "" },
|
|
118
|
+
padTokenId: 179_935,
|
|
119
|
+
inputs: ["input_ids", "attention_mask"],
|
|
120
|
+
output: "last_hidden_state",
|
|
121
|
+
languages:
|
|
122
|
+
"52 languages with retrieval training (German, English, French, Spanish, Italian, Dutch, Polish, …), 200+ in pretraining",
|
|
123
|
+
quantization: "dynamic int8 (uint8 weights), as published",
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
/** Every approved model. */
|
|
127
|
+
export const EMBEDDING_MODELS: readonly EmbeddingModel[] = [GRANITE_97M_R2];
|
|
128
|
+
|
|
129
|
+
/** Catalog ids, for config validation. */
|
|
130
|
+
export const EMBEDDING_MODEL_IDS: readonly string[] = EMBEDDING_MODELS.map((m) => m.id);
|
|
131
|
+
|
|
132
|
+
/** The model a plugin's config offers by default. */
|
|
133
|
+
export const DEFAULT_EMBEDDING_MODEL_ID: string = GRANITE_97M_R2.id;
|
|
134
|
+
|
|
135
|
+
/** The catalog entry of `id`, or undefined. */
|
|
136
|
+
export function embeddingModel(id: string): EmbeddingModel | undefined {
|
|
137
|
+
return EMBEDDING_MODELS.find((m) => m.id === id);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* What an index records as its vector model: the id and the graph's digest.
|
|
142
|
+
* A new revision of the same id is another fingerprint, so an index built
|
|
143
|
+
* with the old files is rebuilt rather than mixed with new query vectors.
|
|
144
|
+
*/
|
|
145
|
+
export function embeddingModelFingerprint(model: EmbeddingModel): string {
|
|
146
|
+
return `${model.id}@${model.files.model.sha256.slice(0, 12)}`;
|
|
147
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
export const PROJECT_NAME = "caelo-cms";
|
|
4
4
|
|
|
5
|
+
export * from "./ai-cost.js";
|
|
5
6
|
export * from "./ai-tools.js";
|
|
6
7
|
export * from "./auth-forms.js";
|
|
7
8
|
export * from "./auto-update.js";
|
|
@@ -24,6 +25,7 @@ export * from "./database-url.js";
|
|
|
24
25
|
export * from "./design-draft-shell.js";
|
|
25
26
|
export * from "./document-language.js";
|
|
26
27
|
export * from "./edit-markers.js";
|
|
28
|
+
export * from "./embedding-models.js";
|
|
27
29
|
export * from "./escaped-line-breaks.js";
|
|
28
30
|
export * from "./font-assets.js";
|
|
29
31
|
export * from "./font-axis-policy.js";
|
|
@@ -57,6 +59,11 @@ export {
|
|
|
57
59
|
export * from "./markup-structure.js";
|
|
58
60
|
export * from "./mcp-access.js";
|
|
59
61
|
export * from "./media.js";
|
|
62
|
+
export {
|
|
63
|
+
CONTENT_INSTANCE_ATTRIBUTE,
|
|
64
|
+
stripContentInstances,
|
|
65
|
+
tagContentInstance,
|
|
66
|
+
} from "./module-id-tag.js";
|
|
60
67
|
export * from "./nested-module-render.js";
|
|
61
68
|
export * from "./owner-setup-token.js";
|
|
62
69
|
export * from "./page-log.js";
|
|
@@ -84,6 +91,7 @@ export * from "./result.js";
|
|
|
84
91
|
export * from "./run-batched.js";
|
|
85
92
|
export * from "./safe-keys.js";
|
|
86
93
|
export * from "./screenshot-capture.js";
|
|
94
|
+
export * from "./search-feed.js";
|
|
87
95
|
export * from "./seo.js";
|
|
88
96
|
export * from "./session-cookie.js";
|
|
89
97
|
export * from "./site-security-headers.js";
|
|
@@ -122,3 +130,4 @@ export {
|
|
|
122
130
|
isNewerVersion,
|
|
123
131
|
parseVersion,
|
|
124
132
|
} from "./version.js";
|
|
133
|
+
export * from "./visitor-ai-origin.js";
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MPL-2.0
|
|
2
2
|
|
|
3
3
|
import { describe, expect, it } from "bun:test";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
CONTENT_INSTANCE_ATTRIBUTE,
|
|
6
|
+
stripContentInstances,
|
|
7
|
+
tagContentInstance,
|
|
8
|
+
tagModuleId,
|
|
9
|
+
} from "./module-id-tag.js";
|
|
5
10
|
|
|
6
11
|
describe("tagModuleId", () => {
|
|
7
12
|
it("stamps the first opening tag", () => {
|
|
@@ -36,3 +41,24 @@ describe("tagModuleId", () => {
|
|
|
36
41
|
expect(performance.now() - start).toBeLessThan(500);
|
|
37
42
|
});
|
|
38
43
|
});
|
|
44
|
+
|
|
45
|
+
describe("the content-instance stamp (epic #1644)", () => {
|
|
46
|
+
const ID = "11111111-1111-4111-8111-111111111111";
|
|
47
|
+
|
|
48
|
+
it("sits next to the module id, once, also on a self-closing root", () => {
|
|
49
|
+
const tagged = tagContentInstance(tagModuleId("<p>hi</p>", "m1"), ID);
|
|
50
|
+
expect(tagged).toBe(
|
|
51
|
+
`<p data-caelo-module-id="m1" ${CONTENT_INSTANCE_ATTRIBUTE}="${ID}">hi</p>`,
|
|
52
|
+
);
|
|
53
|
+
expect(tagContentInstance(tagged, ID)).toBe(tagged);
|
|
54
|
+
expect(tagContentInstance('<line x1="0"/>', ID)).toBe(
|
|
55
|
+
`<line x1="0" ${CONTENT_INSTANCE_ATTRIBUTE}="${ID}"/>`,
|
|
56
|
+
);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it("leaves no trace once stripped", () => {
|
|
60
|
+
const html = `<div data-caelo-module-id="m1" ${CONTENT_INSTANCE_ATTRIBUTE}="${ID}"><p>x</p></div>`;
|
|
61
|
+
expect(stripContentInstances(html)).toBe('<div data-caelo-module-id="m1"><p>x</p></div>');
|
|
62
|
+
expect(stripContentInstances("<p>none</p>")).toBe("<p>none</p>");
|
|
63
|
+
});
|
|
64
|
+
});
|
package/src/module-id-tag.ts
CHANGED
|
@@ -30,18 +30,45 @@ export const MODULE_ID_ATTRIBUTE = "data-caelo-module-id";
|
|
|
30
30
|
* @example tagModuleId('<line x1="0"/>', "abc") // '<line x1="0" data-caelo-module-id="abc"/>'
|
|
31
31
|
*/
|
|
32
32
|
export function tagModuleId(html: string, moduleId: string): string {
|
|
33
|
+
return tagRoot(html, MODULE_ID_ATTRIBUTE, moduleId);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Epic #1644 — the content instance a placement shows, on its root next to
|
|
38
|
+
* the module id, for page-output observers only: two placements of one
|
|
39
|
+
* design with different content are two blocks to an index. The generator
|
|
40
|
+
* stamps it when an observer reads the page and strips it
|
|
41
|
+
* ({@link stripContentInstances}) before the file is written, so the
|
|
42
|
+
* published HTML never carries it.
|
|
43
|
+
*/
|
|
44
|
+
export const CONTENT_INSTANCE_ATTRIBUTE = "data-caelo-content-instance";
|
|
45
|
+
|
|
46
|
+
/** {@link CONTENT_INSTANCE_ATTRIBUTE} on the module's root (idempotent, like {@link tagModuleId}). */
|
|
47
|
+
export function tagContentInstance(html: string, contentInstanceId: string): string {
|
|
48
|
+
return tagRoot(html, CONTENT_INSTANCE_ATTRIBUTE, contentInstanceId);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const STAMP = new RegExp(` ${CONTENT_INSTANCE_ATTRIBUTE}="[0-9a-f-]{36}"`, "g");
|
|
52
|
+
|
|
53
|
+
/** The HTML without the observers' content-instance stamps (a fixed-width pattern: linear). */
|
|
54
|
+
export function stripContentInstances(html: string): string {
|
|
55
|
+
return html.includes(CONTENT_INSTANCE_ATTRIBUTE) ? html.replace(STAMP, "") : html;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** `name="value"` on the first opening tag of `html`, unless that tag has `name` already. */
|
|
59
|
+
function tagRoot(html: string, name: string, value: string): string {
|
|
33
60
|
if (!html) return html;
|
|
34
61
|
const firstOpen = /<([a-zA-Z][a-zA-Z0-9-]*)\b([^>]*)>/;
|
|
35
62
|
const m = firstOpen.exec(html);
|
|
36
63
|
if (!m) return html;
|
|
37
64
|
const tagAttrs = m[2] ?? "";
|
|
38
|
-
if (
|
|
65
|
+
if (new RegExp(`\\s${name}\\s*=`).test(tagAttrs)) return html;
|
|
39
66
|
// String ops, not a `\s*\/\s*$` regex: that backtracks polynomially on
|
|
40
67
|
// long whitespace runs (CodeQL js/polynomial-redos).
|
|
41
68
|
const trimmed = tagAttrs.trimEnd();
|
|
42
69
|
const selfClosing = trimmed.endsWith("/");
|
|
43
70
|
const attrs = selfClosing ? trimmed.slice(0, -1).trimEnd() : tagAttrs;
|
|
44
71
|
const close = selfClosing ? "/>" : ">";
|
|
45
|
-
const replaced = `<${m[1]}${attrs} ${
|
|
72
|
+
const replaced = `<${m[1]}${attrs} ${name}="${value}"${close}`;
|
|
46
73
|
return html.slice(0, m.index) + replaced + html.slice(m.index + m[0].length);
|
|
47
74
|
}
|
|
@@ -60,6 +60,27 @@ describe("composePagePreview", () => {
|
|
|
60
60
|
);
|
|
61
61
|
});
|
|
62
62
|
|
|
63
|
+
it("stamps each placement's content instance only when an observer asked for it (#1644)", () => {
|
|
64
|
+
const input = (stampContentInstances: boolean) => ({
|
|
65
|
+
templateHtml: `<body><caelo-slot name="x">_</caelo-slot></body>`,
|
|
66
|
+
templateCss: "",
|
|
67
|
+
stampContentInstances,
|
|
68
|
+
blocks: [
|
|
69
|
+
{
|
|
70
|
+
blockName: "x",
|
|
71
|
+
modules: [
|
|
72
|
+
{ ...blankModule, html: "<p>A</p>", contentInstanceId: uid(1) },
|
|
73
|
+
{ ...blankModule, html: "<p>B</p>", contentInstanceId: uid(2) },
|
|
74
|
+
],
|
|
75
|
+
},
|
|
76
|
+
],
|
|
77
|
+
});
|
|
78
|
+
const stamped = composePagePreview(input(true)).html;
|
|
79
|
+
expect(stamped).toContain(`data-caelo-content-instance="${uid(1)}">A</p>`);
|
|
80
|
+
expect(stamped).toContain(`data-caelo-content-instance="${uid(2)}">B</p>`);
|
|
81
|
+
expect(composePagePreview(input(false)).html).not.toContain("data-caelo-content-instance");
|
|
82
|
+
});
|
|
83
|
+
|
|
63
84
|
it("orders modules within a block by array order", () => {
|
|
64
85
|
const out = composePagePreview({
|
|
65
86
|
templateHtml: `<body><caelo-slot name="x">_</caelo-slot></body>`,
|
package/src/preview-compose.ts
CHANGED
|
@@ -30,7 +30,7 @@ import type { ModuleFieldKind } from "./content.js";
|
|
|
30
30
|
import { findInteractionIds, interactionAssets } from "./interactions.js";
|
|
31
31
|
import { type LinkResolver, resolveLinkRefsInHtml } from "./link-refs.js";
|
|
32
32
|
import type { ListPageContext } from "./lists/resolve.js";
|
|
33
|
-
import { MODULE_ID_ATTRIBUTE, tagModuleId } from "./module-id-tag.js";
|
|
33
|
+
import { MODULE_ID_ATTRIBUTE, tagContentInstance, tagModuleId } from "./module-id-tag.js";
|
|
34
34
|
import {
|
|
35
35
|
type NestedModuleResource,
|
|
36
36
|
type NestedRenderFailure,
|
|
@@ -137,6 +137,17 @@ export interface ComposeTheme {
|
|
|
137
137
|
};
|
|
138
138
|
}
|
|
139
139
|
|
|
140
|
+
/** The placement's content instance on its root, when an observer reads the page. */
|
|
141
|
+
function stampPlacement(
|
|
142
|
+
html: string,
|
|
143
|
+
m: { readonly contentInstanceId?: string },
|
|
144
|
+
input: { readonly stampContentInstances?: boolean },
|
|
145
|
+
): string {
|
|
146
|
+
return input.stampContentInstances && m.contentInstanceId
|
|
147
|
+
? tagContentInstance(html, m.contentInstanceId)
|
|
148
|
+
: html;
|
|
149
|
+
}
|
|
150
|
+
|
|
140
151
|
/**
|
|
141
152
|
* issue #150 — self-hosted web fonts resolved by the caller (font
|
|
142
153
|
* resolver in admin-core; static generator + preview op thread the
|
|
@@ -150,6 +161,12 @@ export interface ComposeFonts {
|
|
|
150
161
|
}
|
|
151
162
|
|
|
152
163
|
export interface ComposeInput {
|
|
164
|
+
/**
|
|
165
|
+
* Epic #1644 — stamp each placement's content instance on its root
|
|
166
|
+
* (module-id-tag.ts CONTENT_INSTANCE_ATTRIBUTE) for page-output
|
|
167
|
+
* observers; the generator strips it before the file is written.
|
|
168
|
+
*/
|
|
169
|
+
readonly stampContentInstances?: boolean;
|
|
153
170
|
readonly templateHtml: string;
|
|
154
171
|
readonly templateCss: string;
|
|
155
172
|
readonly blocks: readonly ComposeBlock[];
|
|
@@ -299,7 +316,7 @@ export function composePagePreview(input: ComposeInput): ComposeOutput {
|
|
|
299
316
|
const rendered = renderModuleFields(m, input);
|
|
300
317
|
recordFailures(moduleFailures, block.blockName, m, rendered.failures);
|
|
301
318
|
nestedAssetIds.push(...rendered.nestedModuleIds);
|
|
302
|
-
return tagModuleId(rendered.html, m.moduleId);
|
|
319
|
+
return stampPlacement(tagModuleId(rendered.html, m.moduleId), m, input);
|
|
303
320
|
});
|
|
304
321
|
const html = renderedModuleHtml.join("\n");
|
|
305
322
|
contentByName.set(block.blockName, html);
|
|
@@ -699,7 +716,18 @@ function extractBodyInner(composedHtml: string): string {
|
|
|
699
716
|
return peeled ?? inner;
|
|
700
717
|
}
|
|
701
718
|
|
|
702
|
-
|
|
719
|
+
/** {@link composePageWithLayout}'s result. */
|
|
720
|
+
export interface ComposeWithLayoutOutput extends ComposeOutput {
|
|
721
|
+
/**
|
|
722
|
+
* The page's own body: the template with the page's modules, as it fills
|
|
723
|
+
* the layout's `content` slot — without the layout's header, footer and
|
|
724
|
+
* other blocks. The static generator fingerprints it to tell whether what
|
|
725
|
+
* the page itself shows changed (docs/dev/search-engine-feed.md).
|
|
726
|
+
*/
|
|
727
|
+
readonly contentHtml: string;
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
export function composePageWithLayout(input: ComposeWithLayoutInput): ComposeWithLayoutOutput {
|
|
703
731
|
// No-fallbacks (CLAUDE.md §2): validate the layout declares a
|
|
704
732
|
// `content` slot up-front, before rendering. The htmlparser2-based
|
|
705
733
|
// walk handles attribute quoting / ordering uniformly; a layout
|
|
@@ -765,7 +793,7 @@ export function composePageWithLayout(input: ComposeWithLayoutInput): ComposeOut
|
|
|
765
793
|
nestedAssetIds.push(id);
|
|
766
794
|
}
|
|
767
795
|
}
|
|
768
|
-
const tagged = tagModuleId(baseHtml, m.moduleId);
|
|
796
|
+
const tagged = stampPlacement(tagModuleId(baseHtml, m.moduleId), m, input);
|
|
769
797
|
if (!deferral) return tagged;
|
|
770
798
|
if (deferral.omit) {
|
|
771
799
|
if (deferral.editorNotice) deferredCss.add(deferral.editorNotice.css);
|
|
@@ -907,5 +935,6 @@ export function composePageWithLayout(input: ComposeWithLayoutInput): ComposeOut
|
|
|
907
935
|
replacedSlots: [...replacedSet],
|
|
908
936
|
missingSlots: [...missingSet],
|
|
909
937
|
moduleFailures,
|
|
938
|
+
contentHtml: innerBody,
|
|
910
939
|
};
|
|
911
940
|
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
import { describe, expect, it } from "bun:test";
|
|
4
|
+
import {
|
|
5
|
+
indexNowKeyFile,
|
|
6
|
+
isPublicSiteHost,
|
|
7
|
+
pageChangeSchema,
|
|
8
|
+
siteFeedLink,
|
|
9
|
+
siteFeedTitle,
|
|
10
|
+
siteFeedUrl,
|
|
11
|
+
} from "./search-feed.js";
|
|
12
|
+
|
|
13
|
+
describe("the site's feed", () => {
|
|
14
|
+
it("lives at /feed.xml and is titled with the host", () => {
|
|
15
|
+
expect(siteFeedUrl("https://www.example.com/")).toBe("https://www.example.com/feed.xml");
|
|
16
|
+
expect(siteFeedTitle("https://www.example.com")).toBe("www.example.com");
|
|
17
|
+
expect(siteFeedLink(true, "https://example.com")).toEqual({
|
|
18
|
+
url: "https://example.com/feed.xml",
|
|
19
|
+
title: "example.com",
|
|
20
|
+
});
|
|
21
|
+
expect(siteFeedLink(false, "https://example.com")).toBeNull();
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it("serves the IndexNow key at the root", () => {
|
|
25
|
+
expect(indexNowKeyFile("abcdef12")).toBe("abcdef12.txt");
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
describe("isPublicSiteHost", () => {
|
|
30
|
+
it("accepts public names", () => {
|
|
31
|
+
expect(isPublicSiteHost("www.example.org")).toBe(true);
|
|
32
|
+
expect(isPublicSiteHost("caelo-cms.com")).toBe(true);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it("refuses names and addresses search engines cannot reach", () => {
|
|
36
|
+
for (const host of [
|
|
37
|
+
"localhost",
|
|
38
|
+
"admin.localhost",
|
|
39
|
+
"intranet",
|
|
40
|
+
"site.local",
|
|
41
|
+
"api.internal",
|
|
42
|
+
"foo.test",
|
|
43
|
+
"example",
|
|
44
|
+
"printer.home.arpa",
|
|
45
|
+
"127.0.0.1",
|
|
46
|
+
"[::1]",
|
|
47
|
+
"",
|
|
48
|
+
]) {
|
|
49
|
+
expect(isPublicSiteHost(host)).toBe(false);
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe("pageChangeSchema", () => {
|
|
55
|
+
const change = {
|
|
56
|
+
pageId: "11111111-1111-4111-8111-111111111111",
|
|
57
|
+
kind: "changed",
|
|
58
|
+
url: "https://example.com/a/",
|
|
59
|
+
fingerprint: "a".repeat(64),
|
|
60
|
+
pluginFingerprint: null,
|
|
61
|
+
title: "A",
|
|
62
|
+
summary: "",
|
|
63
|
+
prevUrl: "https://example.com/a/",
|
|
64
|
+
changedAt: "2026-11-20T12:00:00.000Z",
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
it("accepts a change and a removal", () => {
|
|
68
|
+
expect(pageChangeSchema.safeParse(change).success).toBe(true);
|
|
69
|
+
expect(
|
|
70
|
+
pageChangeSchema.safeParse({
|
|
71
|
+
...change,
|
|
72
|
+
kind: "removed",
|
|
73
|
+
url: null,
|
|
74
|
+
fingerprint: null,
|
|
75
|
+
title: null,
|
|
76
|
+
summary: null,
|
|
77
|
+
}).success,
|
|
78
|
+
).toBe(true);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("refuses a plugin fingerprint on a removed page", () => {
|
|
82
|
+
expect(
|
|
83
|
+
pageChangeSchema.safeParse({
|
|
84
|
+
...change,
|
|
85
|
+
kind: "removed",
|
|
86
|
+
url: null,
|
|
87
|
+
fingerprint: null,
|
|
88
|
+
title: null,
|
|
89
|
+
summary: null,
|
|
90
|
+
pluginFingerprint: "b".repeat(64),
|
|
91
|
+
}).success,
|
|
92
|
+
).toBe(false);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it("refuses a live page without a title or a removed one with one", () => {
|
|
96
|
+
expect(pageChangeSchema.safeParse({ ...change, title: null }).success).toBe(false);
|
|
97
|
+
expect(
|
|
98
|
+
pageChangeSchema.safeParse({
|
|
99
|
+
...change,
|
|
100
|
+
kind: "removed",
|
|
101
|
+
url: null,
|
|
102
|
+
fingerprint: null,
|
|
103
|
+
summary: null,
|
|
104
|
+
}).success,
|
|
105
|
+
).toBe(false);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it("refuses a removal with a URL and a live page without one", () => {
|
|
109
|
+
expect(pageChangeSchema.safeParse({ ...change, kind: "removed" }).success).toBe(false);
|
|
110
|
+
expect(pageChangeSchema.safeParse({ ...change, url: null, fingerprint: null }).success).toBe(
|
|
111
|
+
false,
|
|
112
|
+
);
|
|
113
|
+
expect(pageChangeSchema.safeParse({ ...change, fingerprint: null }).success).toBe(false);
|
|
114
|
+
});
|
|
115
|
+
});
|