@rubytech/create-realagent-code 0.1.614 → 0.1.616
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/payload/platform/lib/aeo-llms-txt-writer/dist/__tests__/write-llms-txt.test.d.ts +2 -0
- package/payload/platform/lib/aeo-llms-txt-writer/dist/__tests__/write-llms-txt.test.d.ts.map +1 -0
- package/payload/platform/lib/aeo-llms-txt-writer/dist/__tests__/write-llms-txt.test.js +291 -0
- package/payload/platform/lib/aeo-llms-txt-writer/dist/__tests__/write-llms-txt.test.js.map +1 -0
- package/payload/platform/lib/aeo-llms-txt-writer/dist/index.d.ts +13 -1
- package/payload/platform/lib/aeo-llms-txt-writer/dist/index.d.ts.map +1 -1
- package/payload/platform/lib/aeo-llms-txt-writer/dist/index.js +127 -18
- package/payload/platform/lib/aeo-llms-txt-writer/dist/index.js.map +1 -1
- package/payload/platform/lib/aeo-llms-txt-writer/src/__tests__/write-llms-txt.test.ts +391 -0
- package/payload/platform/lib/aeo-llms-txt-writer/src/index.ts +156 -19
- package/payload/platform/lib/aeo-llms-txt-writer/vitest.config.ts +10 -0
- package/payload/platform/lib/mcp-spawn-tee/dist/index.d.ts +15 -1
- package/payload/platform/lib/mcp-spawn-tee/dist/index.d.ts.map +1 -1
- package/payload/platform/lib/mcp-spawn-tee/dist/index.js +129 -7
- package/payload/platform/lib/mcp-spawn-tee/dist/index.js.map +1 -1
- package/payload/platform/lib/mcp-spawn-tee/src/__tests__/spawn-tee.test.ts +227 -0
- package/payload/platform/lib/mcp-spawn-tee/src/index.ts +133 -6
- package/payload/platform/plugins/admin/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/admin/mcp/dist/index.js +11 -3
- package/payload/platform/plugins/admin/mcp/dist/index.js.map +1 -1
- package/payload/platform/plugins/admin/mcp/dist/tools/publish-site.d.ts +3 -0
- package/payload/platform/plugins/admin/mcp/dist/tools/publish-site.d.ts.map +1 -1
- package/payload/platform/plugins/admin/mcp/dist/tools/publish-site.js +38 -20
- package/payload/platform/plugins/admin/mcp/dist/tools/publish-site.js.map +1 -1
- package/payload/platform/plugins/admin/skills/platform-architecture/SKILL.md +6 -5
- package/payload/platform/plugins/admin/skills/publish-site/SKILL.md +3 -0
- package/payload/platform/plugins/admin/skills/whats-new/SKILL.md +13 -0
- package/payload/platform/plugins/aeo/PLUGIN.md +1 -1
- package/payload/platform/plugins/aeo/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/aeo/mcp/dist/index.js +17 -10
- package/payload/platform/plugins/aeo/mcp/dist/index.js.map +1 -1
- package/payload/platform/plugins/browser/PLUGIN.md +1 -1
- package/payload/platform/plugins/browser/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/connector/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/contacts/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/dispatch/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/docs/references/aeo.md +4 -3
- package/payload/platform/plugins/docs/references/plugins-guide.md +1 -1
- package/payload/platform/plugins/email/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/filesystem/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/google/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/graph-viewer/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/inventory/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/ledger/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/memory/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/outlook/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/quickbooks/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/replicate/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/scheduling/PLUGIN.md +3 -4
- package/payload/platform/plugins/scheduling/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/scheduling/mcp/dist/index.js +0 -6
- package/payload/platform/plugins/scheduling/mcp/dist/index.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/lib/__tests__/gate-audits.test.js +65 -20
- package/payload/platform/plugins/scheduling/mcp/dist/lib/__tests__/gate-audits.test.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/lib/__tests__/gate-classification.test.js +20 -0
- package/payload/platform/plugins/scheduling/mcp/dist/lib/__tests__/gate-classification.test.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/lib/gate.d.ts +12 -4
- package/payload/platform/plugins/scheduling/mcp/dist/lib/gate.d.ts.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/lib/gate.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/lib/schedule-audit.d.ts.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/lib/schedule-audit.js +39 -6
- package/payload/platform/plugins/scheduling/mcp/dist/lib/schedule-audit.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/scripts/__tests__/gate-dispatch-wiring.test.js +21 -6
- package/payload/platform/plugins/scheduling/mcp/dist/scripts/__tests__/gate-dispatch-wiring.test.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/scripts/check-due-events.js +26 -5
- package/payload/platform/plugins/scheduling/mcp/dist/scripts/check-due-events.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/tools/__tests__/schedule-gate-waiver.test.js +17 -51
- package/payload/platform/plugins/scheduling/mcp/dist/tools/__tests__/schedule-gate-waiver.test.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/tools/__tests__/schedule-gate-write.test.js +6 -4
- package/payload/platform/plugins/scheduling/mcp/dist/tools/__tests__/schedule-gate-write.test.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/tools/__tests__/scheduling-gate.test.js +113 -1
- package/payload/platform/plugins/scheduling/mcp/dist/tools/__tests__/scheduling-gate.test.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/tools/schedule-event.d.ts +8 -18
- package/payload/platform/plugins/scheduling/mcp/dist/tools/schedule-event.d.ts.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/tools/schedule-event.js +14 -27
- package/payload/platform/plugins/scheduling/mcp/dist/tools/schedule-event.js.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/tools/scheduling-gate.d.ts.map +1 -1
- package/payload/platform/plugins/scheduling/mcp/dist/tools/scheduling-gate.js +125 -56
- package/payload/platform/plugins/scheduling/mcp/dist/tools/scheduling-gate.js.map +1 -1
- package/payload/platform/plugins/storage-broker/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/url-get/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/voice-mirror/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/whatsapp/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/work/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/plugins/workflows/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/platform/scripts/__tests__/vnc-cdp-liveness.test.sh +270 -19
- package/payload/platform/scripts/vnc.sh +174 -71
- package/payload/platform/services/claude-session-manager/dist/index.js +17 -0
- package/payload/platform/services/claude-session-manager/dist/index.js.map +1 -1
- package/payload/platform/services/claude-session-manager/dist/mcp-helper-census.d.ts +80 -0
- package/payload/platform/services/claude-session-manager/dist/mcp-helper-census.d.ts.map +1 -0
- package/payload/platform/services/claude-session-manager/dist/mcp-helper-census.js +201 -0
- package/payload/platform/services/claude-session-manager/dist/mcp-helper-census.js.map +1 -0
- package/payload/premium-plugins/real-agent/plugins/buyers/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/premium-plugins/real-agent/plugins/estate-sales/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/premium-plugins/real-agent/plugins/loop/PLUGIN.md +12 -0
- package/payload/premium-plugins/real-agent/plugins/loop/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/index.js +13 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/index.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/field-census.d.ts +22 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/field-census.d.ts.map +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/field-census.js +81 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/field-census.js.map +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/loop-api.d.ts +17 -5
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/loop-api.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/loop-api.js +90 -17
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/loop-api.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/loop-schema-keys.d.ts +2 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/loop-schema-keys.d.ts.map +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/loop-schema-keys.js +1010 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/lib/loop-schema-keys.js.map +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/auto-responder.js +2 -2
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/auto-responder.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/customer-preferences.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/customer-preferences.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/feedback.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/feedback.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/feedback.js +5 -5
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/feedback.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/marketing-match-batch.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/marketing-match-batch.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/marketing-match-batch.js +2 -2
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/marketing-match-batch.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/marketing-match.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/marketing-match.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/marketing-match.js +4 -4
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/marketing-match.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/people-detail.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/people-detail.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/people-detail.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/people-detail.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/people-search.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/people-search.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/people-search.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/people-search.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-detail.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-detail.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-detail.js +13 -6
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-detail.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-listed.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-listed.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-listed.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-listed.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-search.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-search.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-search.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-search.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-sold.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-sold.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-sold.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/property-sold.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/supplier.js +4 -4
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/supplier.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/team-availability.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/team-availability.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/team-availability.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/team-availability.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/team-info.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/team-info.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/team-info.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/team-info.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/viewing-detail.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/viewing-detail.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/viewing-detail.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/viewing-detail.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/viewing-search.d.ts +1 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/viewing-search.d.ts.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/viewing-search.js +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/dist/tools/viewing-search.js.map +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/package.json +2 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/scripts/generate-loop-schema-keys.mjs +87 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/__tests__/new-tool-shape.test.ts +1 -1
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/__tests__/read-fidelity.test.ts +303 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/__tests__/swagger-write-coverage.test.ts +6 -6
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/index.ts +13 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/lib/field-census.ts +86 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/lib/loop-api.ts +116 -17
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/lib/loop-schema-keys.ts +1010 -0
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/auto-responder.ts +2 -2
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/customer-preferences.ts +3 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/feedback.ts +6 -4
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/marketing-match-batch.ts +4 -2
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/marketing-match.ts +6 -4
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/people-detail.ts +5 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/people-search.ts +5 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/property-detail.ts +17 -7
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/property-listed.ts +5 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/property-search.ts +5 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/property-sold.ts +5 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/supplier.ts +4 -4
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/team-availability.ts +5 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/team-info.ts +6 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/viewing-detail.ts +5 -3
- package/payload/premium-plugins/real-agent/plugins/loop/mcp/src/tools/viewing-search.ts +5 -3
- package/payload/premium-plugins/real-agent/plugins/property-data/lib/mcp-spawn-tee/index.js +129 -7
- package/payload/server/server.js +422 -251
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { mkdtemp, readFile, writeFile } from "node:fs/promises";
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import type { Session } from "neo4j-driver";
|
|
7
|
+
import { writeLlmsTxt } from "../index.js";
|
|
8
|
+
|
|
9
|
+
interface FakeDoc {
|
|
10
|
+
title: string | null;
|
|
11
|
+
url: string | null;
|
|
12
|
+
summary: string | null;
|
|
13
|
+
body?: string | null;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface FakeSection {
|
|
17
|
+
url: string;
|
|
18
|
+
title: string | null;
|
|
19
|
+
body: string | null;
|
|
20
|
+
position: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The lib runs two queries: a flat document query, then a section query
|
|
25
|
+
* scoped to the URLs that survived filtering. The fake dispatches on
|
|
26
|
+
* whether the Cypher mentions HAS_SECTION.
|
|
27
|
+
*/
|
|
28
|
+
function fakeSession(docs: FakeDoc[], sections: FakeSection[] = []) {
|
|
29
|
+
const queries: string[] = [];
|
|
30
|
+
let closed = false;
|
|
31
|
+
const session = {
|
|
32
|
+
async run(cypher: string, params: Record<string, unknown>) {
|
|
33
|
+
queries.push(cypher);
|
|
34
|
+
if (cypher.includes("HAS_SECTION")) {
|
|
35
|
+
const urls = (params.urls as string[]) ?? [];
|
|
36
|
+
const rows = sections
|
|
37
|
+
.filter((s) => urls.includes(s.url))
|
|
38
|
+
.sort((a, b) =>
|
|
39
|
+
a.url === b.url ? a.position - b.position : a.url < b.url ? -1 : 1,
|
|
40
|
+
);
|
|
41
|
+
return {
|
|
42
|
+
records: rows.map((s) => ({
|
|
43
|
+
get: (k: string) =>
|
|
44
|
+
k === "url"
|
|
45
|
+
? s.url
|
|
46
|
+
: k === "title"
|
|
47
|
+
? s.title
|
|
48
|
+
: k === "body"
|
|
49
|
+
? (s.body ?? "")
|
|
50
|
+
: s.position,
|
|
51
|
+
})),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
const rows = [...docs].sort((a, b) =>
|
|
55
|
+
(a.title ?? "") < (b.title ?? "") ? -1 : 1,
|
|
56
|
+
);
|
|
57
|
+
return {
|
|
58
|
+
records: rows.map((d) => ({
|
|
59
|
+
get: (k: string) =>
|
|
60
|
+
k === "title"
|
|
61
|
+
? d.title
|
|
62
|
+
: k === "url"
|
|
63
|
+
? d.url
|
|
64
|
+
: k === "summary"
|
|
65
|
+
? d.summary
|
|
66
|
+
: (d.body ?? ""),
|
|
67
|
+
})),
|
|
68
|
+
};
|
|
69
|
+
},
|
|
70
|
+
async close() {
|
|
71
|
+
closed = true;
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
return {
|
|
75
|
+
deps: { getSession: () => session as unknown as Session },
|
|
76
|
+
queries,
|
|
77
|
+
wasClosed: () => closed,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
describe("writeLlmsTxt — site scoping", () => {
|
|
82
|
+
it("includes only documents whose url starts with siteOrigin", async () => {
|
|
83
|
+
const f = fakeSession([
|
|
84
|
+
{ title: "Mine", url: "https://a.example/x", summary: "page a" },
|
|
85
|
+
{ title: "Theirs", url: "https://b.example/y", summary: "page b" },
|
|
86
|
+
]);
|
|
87
|
+
const r = await writeLlmsTxt(
|
|
88
|
+
{
|
|
89
|
+
accountId: "acct",
|
|
90
|
+
siteName: "A",
|
|
91
|
+
siteOrigin: "https://a.example",
|
|
92
|
+
},
|
|
93
|
+
f.deps,
|
|
94
|
+
);
|
|
95
|
+
expect(r.pageCount).toBe(1);
|
|
96
|
+
expect(r.skippedOtherSite).toBe(1);
|
|
97
|
+
expect(r.llmsTxt).toContain("https://a.example/x");
|
|
98
|
+
expect(r.llmsTxt).not.toContain("b.example");
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it("still counts documents missing a url or title separately", async () => {
|
|
102
|
+
const f = fakeSession([
|
|
103
|
+
{ title: "Mine", url: "https://a.example/x", summary: "page a" },
|
|
104
|
+
{ title: "No url", url: null, summary: "orphan" },
|
|
105
|
+
{ title: null, url: "https://a.example/z", summary: "no title" },
|
|
106
|
+
{ title: "Theirs", url: "https://b.example/y", summary: "page b" },
|
|
107
|
+
]);
|
|
108
|
+
const r = await writeLlmsTxt(
|
|
109
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
110
|
+
f.deps,
|
|
111
|
+
);
|
|
112
|
+
expect(r.pageCount).toBe(1);
|
|
113
|
+
expect(r.skippedNoUrl).toBe(2);
|
|
114
|
+
expect(r.skippedOtherSite).toBe(1);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it("normalises a trailing slash off the origin", async () => {
|
|
118
|
+
const f = fakeSession([
|
|
119
|
+
{ title: "Mine", url: "https://a.example/x", summary: "page a" },
|
|
120
|
+
]);
|
|
121
|
+
const r = await writeLlmsTxt(
|
|
122
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example/" },
|
|
123
|
+
f.deps,
|
|
124
|
+
);
|
|
125
|
+
expect(r.pageCount).toBe(1);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it("does not match a different host that shares the origin as a prefix", async () => {
|
|
129
|
+
const f = fakeSession([
|
|
130
|
+
{ title: "Mine", url: "https://a.example/x", summary: "page a" },
|
|
131
|
+
{ title: "Lookalike", url: "https://a.example.evil/x", summary: "evil" },
|
|
132
|
+
{ title: "Suffixed", url: "https://a.exampleX/x", summary: "other" },
|
|
133
|
+
{ title: "Bare", url: "https://a.example", summary: "root, no slash" },
|
|
134
|
+
]);
|
|
135
|
+
const r = await writeLlmsTxt(
|
|
136
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
137
|
+
f.deps,
|
|
138
|
+
);
|
|
139
|
+
expect(r.pageCount).toBe(2);
|
|
140
|
+
expect(r.skippedOtherSite).toBe(2);
|
|
141
|
+
expect(r.llmsTxt).not.toContain("a.example.evil");
|
|
142
|
+
expect(r.llmsTxt).not.toContain("a.exampleX");
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("throws on an origin that is not an absolute http(s) URL", async () => {
|
|
146
|
+
const f = fakeSession([]);
|
|
147
|
+
for (const bad of ["maxy.institute", "", "/institute", "ftp://a.example"]) {
|
|
148
|
+
await expect(
|
|
149
|
+
writeLlmsTxt(
|
|
150
|
+
{ accountId: "acct", siteName: "A", siteOrigin: bad },
|
|
151
|
+
f.deps,
|
|
152
|
+
),
|
|
153
|
+
).rejects.toThrow(/siteOrigin must be an absolute http\(s\) URL/);
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
describe("writeLlmsTxt — full text", () => {
|
|
159
|
+
it("renders a page's sections in position order under h2 headings", async () => {
|
|
160
|
+
const f = fakeSession(
|
|
161
|
+
[{ title: "Chapter one", url: "https://a.example/one", summary: "sum" }],
|
|
162
|
+
[
|
|
163
|
+
{
|
|
164
|
+
url: "https://a.example/one",
|
|
165
|
+
title: "Third",
|
|
166
|
+
body: "gamma",
|
|
167
|
+
position: 2,
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
url: "https://a.example/one",
|
|
171
|
+
title: "First",
|
|
172
|
+
body: "alpha",
|
|
173
|
+
position: 0,
|
|
174
|
+
},
|
|
175
|
+
{
|
|
176
|
+
url: "https://a.example/one",
|
|
177
|
+
title: "Second",
|
|
178
|
+
body: "beta",
|
|
179
|
+
position: 1,
|
|
180
|
+
},
|
|
181
|
+
],
|
|
182
|
+
);
|
|
183
|
+
const r = await writeLlmsTxt(
|
|
184
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
185
|
+
f.deps,
|
|
186
|
+
);
|
|
187
|
+
const alpha = r.llmsFullTxt.indexOf("alpha");
|
|
188
|
+
const beta = r.llmsFullTxt.indexOf("beta");
|
|
189
|
+
const gamma = r.llmsFullTxt.indexOf("gamma");
|
|
190
|
+
expect(alpha).toBeGreaterThan(-1);
|
|
191
|
+
expect(alpha).toBeLessThan(beta);
|
|
192
|
+
expect(beta).toBeLessThan(gamma);
|
|
193
|
+
expect(r.llmsFullTxt).toContain("## First");
|
|
194
|
+
// Not a size comparison: on a small fixture the index legitimately
|
|
195
|
+
// exceeds the full file. What discriminates is that the section prose
|
|
196
|
+
// reaches llms-full.txt and never reaches the index.
|
|
197
|
+
expect(r.pagesWithFullText).toBe(1);
|
|
198
|
+
expect(r.sectionCount).toBe(3);
|
|
199
|
+
expect(r.llmsTxt).not.toContain("alpha");
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it("falls back to the document body when the page has no sections", async () => {
|
|
203
|
+
const f = fakeSession([
|
|
204
|
+
{
|
|
205
|
+
title: "Standalone",
|
|
206
|
+
url: "https://a.example/s",
|
|
207
|
+
summary: "sum",
|
|
208
|
+
body: "the whole document body",
|
|
209
|
+
},
|
|
210
|
+
]);
|
|
211
|
+
const r = await writeLlmsTxt(
|
|
212
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
213
|
+
f.deps,
|
|
214
|
+
);
|
|
215
|
+
expect(r.llmsFullTxt).toContain("the whole document body");
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
it("emits the summary and counts the page when it has neither sections nor a body", async () => {
|
|
219
|
+
const f = fakeSession([
|
|
220
|
+
{ title: "Thin", url: "https://a.example/t", summary: "only a summary" },
|
|
221
|
+
]);
|
|
222
|
+
const r = await writeLlmsTxt(
|
|
223
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
224
|
+
f.deps,
|
|
225
|
+
);
|
|
226
|
+
expect(r.llmsFullTxt).toContain("only a summary");
|
|
227
|
+
expect(r.pagesWithFullText).toBe(0);
|
|
228
|
+
expect(r.pagesWithoutFullText).toBe(1);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it("demotes every in-body h1, not just the first", async () => {
|
|
232
|
+
const f = fakeSession(
|
|
233
|
+
[{ title: "Multi", url: "https://a.example/m", summary: "sum" }],
|
|
234
|
+
[
|
|
235
|
+
{
|
|
236
|
+
url: "https://a.example/m",
|
|
237
|
+
title: "Body",
|
|
238
|
+
body: "# one\ntext\n# two\nmore",
|
|
239
|
+
position: 0,
|
|
240
|
+
},
|
|
241
|
+
],
|
|
242
|
+
);
|
|
243
|
+
const r = await writeLlmsTxt(
|
|
244
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
245
|
+
f.deps,
|
|
246
|
+
);
|
|
247
|
+
expect(r.llmsFullTxt).toContain("## one");
|
|
248
|
+
expect(r.llmsFullTxt).toContain("## two");
|
|
249
|
+
expect(r.llmsFullTxt).not.toMatch(/^# one$/m);
|
|
250
|
+
expect(r.llmsFullTxt).not.toMatch(/^# two$/m);
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
it("never emits abstract as full text", async () => {
|
|
254
|
+
// The document query no longer reads d.abstract at all: the fake
|
|
255
|
+
// returns only what the query asks for, and body is empty here.
|
|
256
|
+
const f = fakeSession([
|
|
257
|
+
{
|
|
258
|
+
title: "Abstracted",
|
|
259
|
+
url: "https://a.example/a",
|
|
260
|
+
summary: "sum",
|
|
261
|
+
body: "",
|
|
262
|
+
},
|
|
263
|
+
]);
|
|
264
|
+
const r = await writeLlmsTxt(
|
|
265
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
266
|
+
f.deps,
|
|
267
|
+
);
|
|
268
|
+
expect(r.pagesWithoutFullText).toBe(1);
|
|
269
|
+
const docQuery = f.queries.find((q) => !q.includes("HAS_SECTION")) ?? "";
|
|
270
|
+
expect(docQuery).not.toContain("abstract");
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
it("gives every document sharing a url its own sections", async () => {
|
|
274
|
+
const f = fakeSession(
|
|
275
|
+
[
|
|
276
|
+
{ title: "First copy", url: "https://a.example/dup", summary: "one" },
|
|
277
|
+
{ title: "Second copy", url: "https://a.example/dup", summary: "two" },
|
|
278
|
+
],
|
|
279
|
+
[
|
|
280
|
+
{
|
|
281
|
+
url: "https://a.example/dup",
|
|
282
|
+
title: "Body",
|
|
283
|
+
body: "shared prose",
|
|
284
|
+
position: 0,
|
|
285
|
+
},
|
|
286
|
+
],
|
|
287
|
+
);
|
|
288
|
+
const r = await writeLlmsTxt(
|
|
289
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
290
|
+
f.deps,
|
|
291
|
+
);
|
|
292
|
+
expect(r.pageCount).toBe(2);
|
|
293
|
+
expect(r.pagesWithFullText).toBe(2);
|
|
294
|
+
expect(r.pagesWithoutFullText).toBe(0);
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
it("escapes a section title that carries a newline", async () => {
|
|
298
|
+
const f = fakeSession(
|
|
299
|
+
[{ title: "Injected", url: "https://a.example/i", summary: "sum" }],
|
|
300
|
+
[
|
|
301
|
+
{
|
|
302
|
+
url: "https://a.example/i",
|
|
303
|
+
title: "Part one\n# Overview",
|
|
304
|
+
body: "prose",
|
|
305
|
+
position: 0,
|
|
306
|
+
},
|
|
307
|
+
],
|
|
308
|
+
);
|
|
309
|
+
const r = await writeLlmsTxt(
|
|
310
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
311
|
+
f.deps,
|
|
312
|
+
);
|
|
313
|
+
expect(r.llmsFullTxt).toContain("## Part one # Overview");
|
|
314
|
+
// Exactly one H1 per page block: the page title itself.
|
|
315
|
+
expect(r.llmsFullTxt.match(/^# /gm)?.length).toBe(2); // site header + page
|
|
316
|
+
});
|
|
317
|
+
|
|
318
|
+
it("does not run the section query when no document survives filtering", async () => {
|
|
319
|
+
const f = fakeSession([
|
|
320
|
+
{ title: "Theirs", url: "https://b.example/y", summary: "page b" },
|
|
321
|
+
]);
|
|
322
|
+
await writeLlmsTxt(
|
|
323
|
+
{ accountId: "acct", siteName: "A", siteOrigin: "https://a.example" },
|
|
324
|
+
f.deps,
|
|
325
|
+
);
|
|
326
|
+
expect(f.queries.some((q) => q.includes("HAS_SECTION"))).toBe(false);
|
|
327
|
+
});
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
describe("writeLlmsTxt — refusing an empty full-text file", () => {
|
|
331
|
+
it("writes llms.txt but not llms-full.txt when no page has full text", async () => {
|
|
332
|
+
const dir = await mkdtemp(join(tmpdir(), "aeo-"));
|
|
333
|
+
await writeFile(join(dir, "llms-full.txt"), "PRIOR GOOD FILE", "utf-8");
|
|
334
|
+
const f = fakeSession([
|
|
335
|
+
{ title: "Thin", url: "https://a.example/t", summary: "only a summary" },
|
|
336
|
+
]);
|
|
337
|
+
const r = await writeLlmsTxt(
|
|
338
|
+
{
|
|
339
|
+
accountId: "acct",
|
|
340
|
+
siteName: "A",
|
|
341
|
+
siteOrigin: "https://a.example",
|
|
342
|
+
siteDir: dir,
|
|
343
|
+
},
|
|
344
|
+
f.deps,
|
|
345
|
+
);
|
|
346
|
+
expect(r.writtenPaths?.index).toBe(join(dir, "llms.txt"));
|
|
347
|
+
expect(r.writtenPaths?.full).toBeNull();
|
|
348
|
+
expect(r.fullRefusedReason).toBe("no-full-text-any-page");
|
|
349
|
+
expect(await readFile(join(dir, "llms-full.txt"), "utf-8")).toBe(
|
|
350
|
+
"PRIOR GOOD FILE",
|
|
351
|
+
);
|
|
352
|
+
expect(existsSync(join(dir, "llms.txt"))).toBe(true);
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
it("writes both files when at least one page has full text", async () => {
|
|
356
|
+
const dir = await mkdtemp(join(tmpdir(), "aeo-"));
|
|
357
|
+
const f = fakeSession(
|
|
358
|
+
[
|
|
359
|
+
{
|
|
360
|
+
title: "Thin",
|
|
361
|
+
url: "https://a.example/t",
|
|
362
|
+
summary: "only a summary",
|
|
363
|
+
},
|
|
364
|
+
{ title: "Full", url: "https://a.example/f", summary: "sum" },
|
|
365
|
+
],
|
|
366
|
+
[
|
|
367
|
+
{
|
|
368
|
+
url: "https://a.example/f",
|
|
369
|
+
title: "Body",
|
|
370
|
+
body: "real prose",
|
|
371
|
+
position: 0,
|
|
372
|
+
},
|
|
373
|
+
],
|
|
374
|
+
);
|
|
375
|
+
const r = await writeLlmsTxt(
|
|
376
|
+
{
|
|
377
|
+
accountId: "acct",
|
|
378
|
+
siteName: "A",
|
|
379
|
+
siteOrigin: "https://a.example",
|
|
380
|
+
siteDir: dir,
|
|
381
|
+
},
|
|
382
|
+
f.deps,
|
|
383
|
+
);
|
|
384
|
+
expect(r.writtenPaths?.full).toBe(join(dir, "llms-full.txt"));
|
|
385
|
+
expect(r.fullRefusedReason).toBeNull();
|
|
386
|
+
expect(await readFile(join(dir, "llms-full.txt"), "utf-8")).toContain(
|
|
387
|
+
"real prose",
|
|
388
|
+
);
|
|
389
|
+
expect(r.pagesWithoutFullText).toBe(1);
|
|
390
|
+
});
|
|
391
|
+
});
|
|
@@ -13,6 +13,13 @@ import type { Session } from "neo4j-driver";
|
|
|
13
13
|
export interface WriteLlmsTxtInput {
|
|
14
14
|
accountId: string;
|
|
15
15
|
siteName: string;
|
|
16
|
+
/**
|
|
17
|
+
* Absolute http(s) origin of the site being generated, e.g.
|
|
18
|
+
* "https://maxy.institute". Only KnowledgeDocuments whose url starts
|
|
19
|
+
* with this origin are included. Required: an account publishes many
|
|
20
|
+
* sites, and an account-wide llms.txt is never correct for any of them.
|
|
21
|
+
*/
|
|
22
|
+
siteOrigin: string;
|
|
16
23
|
siteDescription?: string;
|
|
17
24
|
siteDir?: string;
|
|
18
25
|
}
|
|
@@ -26,8 +33,18 @@ export interface WriteLlmsTxtResult {
|
|
|
26
33
|
llmsFullTxt: string;
|
|
27
34
|
pageCount: number;
|
|
28
35
|
skippedNoUrl: number;
|
|
36
|
+
skippedOtherSite: number;
|
|
37
|
+
pagesWithFullText: number;
|
|
38
|
+
pagesWithoutFullText: number;
|
|
39
|
+
sectionCount: number;
|
|
29
40
|
sizeBytes: { index: number; full: number };
|
|
30
|
-
|
|
41
|
+
fullRefusedReason: string | null;
|
|
42
|
+
writtenPaths?: { index: string; full: string | null };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
interface SectionRow {
|
|
46
|
+
title: string;
|
|
47
|
+
body: string;
|
|
31
48
|
}
|
|
32
49
|
|
|
33
50
|
interface PageRow {
|
|
@@ -35,52 +52,143 @@ interface PageRow {
|
|
|
35
52
|
url: string;
|
|
36
53
|
summary: string;
|
|
37
54
|
body: string;
|
|
55
|
+
sections: SectionRow[];
|
|
38
56
|
}
|
|
39
57
|
|
|
40
|
-
const HEADER_RX = /^#\s/
|
|
58
|
+
const HEADER_RX = /^#\s/gm;
|
|
41
59
|
|
|
42
60
|
function escapeMarkdown(text: string): string {
|
|
43
61
|
return text.replace(/[\[\]\(\)\n\r]/g, " ").trim();
|
|
44
62
|
}
|
|
45
63
|
|
|
64
|
+
function normaliseOrigin(raw: string): string {
|
|
65
|
+
let parsed: URL;
|
|
66
|
+
try {
|
|
67
|
+
parsed = new URL(raw);
|
|
68
|
+
} catch {
|
|
69
|
+
throw new Error(`siteOrigin must be an absolute http(s) URL: ${raw}`);
|
|
70
|
+
}
|
|
71
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
72
|
+
throw new Error(`siteOrigin must be an absolute http(s) URL: ${raw}`);
|
|
73
|
+
}
|
|
74
|
+
return parsed.origin;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* True when `url` is the origin itself or a path beneath it. A bare
|
|
79
|
+
* startsWith would also admit a different host that merely shares the
|
|
80
|
+
* origin as a string prefix ("https://a.example" vs "https://a.example.evil"),
|
|
81
|
+
* which is the cross-site leak this filter exists to stop.
|
|
82
|
+
*/
|
|
83
|
+
function isUnderOrigin(url: string, origin: string): boolean {
|
|
84
|
+
if (!url.startsWith(origin)) return false;
|
|
85
|
+
const rest = url.slice(origin.length);
|
|
86
|
+
return (
|
|
87
|
+
rest === "" ||
|
|
88
|
+
rest.startsWith("/") ||
|
|
89
|
+
rest.startsWith("?") ||
|
|
90
|
+
rest.startsWith("#")
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
|
|
46
94
|
function summariseBody(body: string): string {
|
|
47
95
|
const flat = body.replace(/\s+/g, " ").trim();
|
|
48
96
|
if (flat.length <= 200) return flat;
|
|
49
97
|
return flat.slice(0, 197).trimEnd() + "…";
|
|
50
98
|
}
|
|
51
99
|
|
|
100
|
+
/**
|
|
101
|
+
* A page's full text. Returns "" when the page carries none of any kind —
|
|
102
|
+
* the single fact the pagesWithoutFullText counter and the write refusal
|
|
103
|
+
* are both derived from.
|
|
104
|
+
*/
|
|
105
|
+
function renderFullBody(row: PageRow): string {
|
|
106
|
+
if (row.sections.length > 0) {
|
|
107
|
+
return row.sections
|
|
108
|
+
.map((s) => {
|
|
109
|
+
const heading = s.title ? `## ${s.title}\n\n` : "";
|
|
110
|
+
return `${heading}${s.body.replace(HEADER_RX, "## ")}`.trim();
|
|
111
|
+
})
|
|
112
|
+
.filter((block) => block.length > 0)
|
|
113
|
+
.join("\n\n");
|
|
114
|
+
}
|
|
115
|
+
if (row.body) return row.body.replace(HEADER_RX, "## ");
|
|
116
|
+
return "";
|
|
117
|
+
}
|
|
118
|
+
|
|
52
119
|
export async function writeLlmsTxt(
|
|
53
120
|
input: WriteLlmsTxtInput,
|
|
54
121
|
deps: WriteLlmsTxtDeps,
|
|
55
122
|
): Promise<WriteLlmsTxtResult> {
|
|
123
|
+
const origin = normaliseOrigin(input.siteOrigin);
|
|
56
124
|
const session = deps.getSession();
|
|
57
|
-
|
|
125
|
+
const rows: PageRow[] = [];
|
|
58
126
|
let skippedNoUrl = 0;
|
|
127
|
+
let skippedOtherSite = 0;
|
|
59
128
|
try {
|
|
60
129
|
const r = await session.run(
|
|
61
130
|
`MATCH (d:KnowledgeDocument {accountId: $accountId})
|
|
62
131
|
RETURN d.title AS title, d.url AS url, d.summary AS summary,
|
|
63
|
-
coalesce(d.body,
|
|
132
|
+
coalesce(d.body, '') AS body
|
|
64
133
|
ORDER BY d.title ASC`,
|
|
65
134
|
{ accountId: input.accountId },
|
|
66
135
|
);
|
|
67
136
|
for (const rec of r.records) {
|
|
68
137
|
const title = (rec.get("title") as string | null) ?? "";
|
|
69
|
-
const url = (rec.get("url") as string | null) ?? "";
|
|
138
|
+
const url = ((rec.get("url") as string | null) ?? "").trim();
|
|
70
139
|
const summary = (rec.get("summary") as string | null) ?? "";
|
|
71
140
|
const body = (rec.get("body") as string | null) ?? "";
|
|
72
141
|
if (!url || !title) {
|
|
73
142
|
skippedNoUrl += 1;
|
|
74
143
|
continue;
|
|
75
144
|
}
|
|
145
|
+
if (!isUnderOrigin(url, origin)) {
|
|
146
|
+
skippedOtherSite += 1;
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
76
149
|
rows.push({
|
|
77
150
|
title: escapeMarkdown(title),
|
|
78
|
-
url
|
|
151
|
+
url,
|
|
79
152
|
summary:
|
|
80
153
|
escapeMarkdown(summary) || summariseBody(body) || "(no summary)",
|
|
81
154
|
body: body.trim(),
|
|
155
|
+
sections: [],
|
|
82
156
|
});
|
|
83
157
|
}
|
|
158
|
+
|
|
159
|
+
// Second query: the full text. Scoped to the URLs that survived
|
|
160
|
+
// filtering so no other site's section bodies are ever read. The prose
|
|
161
|
+
// lives on (:KnowledgeDocument)-[:HAS_SECTION]->(:Section) in `position`
|
|
162
|
+
// order — see platform/plugins/memory/references/schema-base.md.
|
|
163
|
+
if (rows.length > 0) {
|
|
164
|
+
// Keyed to a list, not a single row: two KnowledgeDocuments can carry
|
|
165
|
+
// the same url, and a plain Map would drop one of them silently and
|
|
166
|
+
// then report it as having no full text.
|
|
167
|
+
const byUrl = new Map<string, PageRow[]>();
|
|
168
|
+
for (const row of rows) {
|
|
169
|
+
const bucket = byUrl.get(row.url);
|
|
170
|
+
if (bucket) bucket.push(row);
|
|
171
|
+
else byUrl.set(row.url, [row]);
|
|
172
|
+
}
|
|
173
|
+
const sec = await session.run(
|
|
174
|
+
`MATCH (d:KnowledgeDocument {accountId: $accountId})-[:HAS_SECTION]->(s:Section)
|
|
175
|
+
WHERE d.url IN $urls
|
|
176
|
+
RETURN d.url AS url, s.title AS title,
|
|
177
|
+
coalesce(s.body, '') AS body,
|
|
178
|
+
coalesce(s.position, 0) AS position
|
|
179
|
+
ORDER BY d.url ASC, position ASC`,
|
|
180
|
+
{ accountId: input.accountId, urls: rows.map((row) => row.url) },
|
|
181
|
+
);
|
|
182
|
+
for (const rec of sec.records) {
|
|
183
|
+
const bucket = byUrl.get((rec.get("url") as string) ?? "");
|
|
184
|
+
if (!bucket) continue;
|
|
185
|
+
const section: SectionRow = {
|
|
186
|
+
title: escapeMarkdown((rec.get("title") as string | null) ?? ""),
|
|
187
|
+
body: ((rec.get("body") as string | null) ?? "").trim(),
|
|
188
|
+
};
|
|
189
|
+
for (const row of bucket) row.sections.push(section);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
84
192
|
} finally {
|
|
85
193
|
await session.close();
|
|
86
194
|
}
|
|
@@ -110,21 +218,23 @@ export async function writeLlmsTxt(
|
|
|
110
218
|
fullLines.push(`> ${escapeMarkdown(input.siteDescription)}`);
|
|
111
219
|
fullLines.push("");
|
|
112
220
|
}
|
|
113
|
-
|
|
221
|
+
// Render each page's body once. Rendering once is what lets the counters
|
|
222
|
+
// and the write refusal name the same set of pages.
|
|
223
|
+
const fullBodies = rows.map((row) => renderFullBody(row));
|
|
224
|
+
const pagesWithFullText = fullBodies.filter((b) => b.length > 0).length;
|
|
225
|
+
const pagesWithoutFullText = rows.length - pagesWithFullText;
|
|
226
|
+
const sectionCount = rows.reduce((n, row) => n + row.sections.length, 0);
|
|
227
|
+
|
|
228
|
+
rows.forEach((row, i) => {
|
|
114
229
|
fullLines.push(`# ${row.title}`);
|
|
115
230
|
fullLines.push("");
|
|
116
231
|
fullLines.push(`Source: ${row.url}`);
|
|
117
232
|
fullLines.push("");
|
|
118
|
-
|
|
119
|
-
const safeBody = row.body.replace(HEADER_RX, "## ");
|
|
120
|
-
fullLines.push(safeBody);
|
|
121
|
-
} else {
|
|
122
|
-
fullLines.push(row.summary);
|
|
123
|
-
}
|
|
233
|
+
fullLines.push(fullBodies[i] || row.summary);
|
|
124
234
|
fullLines.push("");
|
|
125
235
|
fullLines.push("---");
|
|
126
236
|
fullLines.push("");
|
|
127
|
-
}
|
|
237
|
+
});
|
|
128
238
|
const llmsFullTxt = fullLines.join("\n");
|
|
129
239
|
|
|
130
240
|
const sizeBytes = {
|
|
@@ -133,10 +243,23 @@ export async function writeLlmsTxt(
|
|
|
133
243
|
};
|
|
134
244
|
|
|
135
245
|
process.stderr.write(
|
|
136
|
-
`[aeo-llms-txt] site=${input.siteName} pages=${rows.length} skippedNoUrl=${skippedNoUrl} indexBytes=${sizeBytes.index} fullBytes=${sizeBytes.full}\n`,
|
|
246
|
+
`[aeo-llms-txt] site=${input.siteName} origin=${origin} pages=${rows.length} skippedNoUrl=${skippedNoUrl} skippedOtherSite=${skippedOtherSite} pagesWithFullText=${pagesWithFullText} pagesWithoutFullText=${pagesWithoutFullText} sections=${sectionCount} indexBytes=${sizeBytes.index} fullBytes=${sizeBytes.full}\n`,
|
|
137
247
|
);
|
|
138
248
|
|
|
139
|
-
|
|
249
|
+
if (pagesWithoutFullText > 0) {
|
|
250
|
+
const missing = rows
|
|
251
|
+
.filter((_, i) => fullBodies[i].length === 0)
|
|
252
|
+
.map((row) => row.url);
|
|
253
|
+
const shown = missing.slice(0, 20);
|
|
254
|
+
process.stderr.write(
|
|
255
|
+
`[aeo-llms-txt] WARN op=full-text-missing pages=${missing.length} shown=${shown.length} urls=[${shown.join(" ")}]\n`,
|
|
256
|
+
);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
const fullRefusedReason =
|
|
260
|
+
pagesWithFullText === 0 ? "no-full-text-any-page" : null;
|
|
261
|
+
|
|
262
|
+
let writtenPaths: { index: string; full: string | null } | undefined;
|
|
140
263
|
if (input.siteDir) {
|
|
141
264
|
const dir = resolvePath(input.siteDir);
|
|
142
265
|
let dirStat;
|
|
@@ -152,12 +275,21 @@ export async function writeLlmsTxt(
|
|
|
152
275
|
throw new Error(`siteDir is not a directory: ${dir}`);
|
|
153
276
|
}
|
|
154
277
|
const indexPath = resolvePath(dir, "llms.txt");
|
|
155
|
-
const fullPath = resolvePath(dir, "llms-full.txt");
|
|
156
278
|
await writeFile(indexPath, llmsTxt, "utf-8");
|
|
157
|
-
|
|
279
|
+
let fullPath: string | null = null;
|
|
280
|
+
if (fullRefusedReason) {
|
|
281
|
+
// A full-text file with no full text in it is worse than a stale
|
|
282
|
+
// one: it silently replaces real prose with the index's summaries.
|
|
283
|
+
process.stderr.write(
|
|
284
|
+
`[aeo-llms-txt] refuse op=write-full reason=${fullRefusedReason} site=${dir}\n`,
|
|
285
|
+
);
|
|
286
|
+
} else {
|
|
287
|
+
fullPath = resolvePath(dir, "llms-full.txt");
|
|
288
|
+
await writeFile(fullPath, llmsFullTxt, "utf-8");
|
|
289
|
+
}
|
|
158
290
|
writtenPaths = { index: indexPath, full: fullPath };
|
|
159
291
|
process.stderr.write(
|
|
160
|
-
`[aeo-llms-txt] wrote site=${dir} indexBytes=${sizeBytes.index} fullBytes=${sizeBytes.full}\n`,
|
|
292
|
+
`[aeo-llms-txt] wrote site=${dir} indexBytes=${sizeBytes.index} fullBytes=${fullPath ? sizeBytes.full : 0}\n`,
|
|
161
293
|
);
|
|
162
294
|
}
|
|
163
295
|
|
|
@@ -166,7 +298,12 @@ export async function writeLlmsTxt(
|
|
|
166
298
|
llmsFullTxt,
|
|
167
299
|
pageCount: rows.length,
|
|
168
300
|
skippedNoUrl,
|
|
301
|
+
skippedOtherSite,
|
|
302
|
+
pagesWithFullText,
|
|
303
|
+
pagesWithoutFullText,
|
|
304
|
+
sectionCount,
|
|
169
305
|
sizeBytes,
|
|
306
|
+
fullRefusedReason,
|
|
170
307
|
writtenPaths,
|
|
171
308
|
};
|
|
172
309
|
}
|