@hraness/kb 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +43 -0
- package/LICENSE +21 -0
- package/README.md +545 -0
- package/dist/agent-context.js +35 -0
- package/dist/agent-guide-audit.js +18 -0
- package/dist/attachments.js +19 -0
- package/dist/authoring.js +33 -0
- package/dist/benchmark.js +19 -0
- package/dist/capture.js +49 -0
- package/dist/cli.js +3646 -0
- package/dist/clip/acquire.js +57 -0
- package/dist/clip/args.js +21 -0
- package/dist/clip/bounded-byte-buffer.js +9 -0
- package/dist/clip/browser-profiles.js +17 -0
- package/dist/clip/bundle-reader.js +11 -0
- package/dist/clip/cli.js +27 -0
- package/dist/clip/cookies.js +21 -0
- package/dist/clip/defuddle-worker.js +84 -0
- package/dist/clip/doctor.js +32 -0
- package/dist/clip/jobs.js +784 -0
- package/dist/clip/network-proxy.js +9 -0
- package/dist/clip/network.js +28 -0
- package/dist/clip/persist.js +30 -0
- package/dist/clip/refresh.js +11 -0
- package/dist/clip/terminal.js +11 -0
- package/dist/evaluation-builder.js +12238 -0
- package/dist/evaluation-kb.js +26 -0
- package/dist/evaluation.js +35 -0
- package/dist/git.js +39 -0
- package/dist/graph.js +39 -0
- package/dist/index-01jj6rbv.js +1103 -0
- package/dist/index-06c9ctr6.js +738 -0
- package/dist/index-0kavxzqj.js +2480 -0
- package/dist/index-1gwbassd.js +743 -0
- package/dist/index-1n418kb9.js +827 -0
- package/dist/index-1vrd1rmn.js +56 -0
- package/dist/index-1xxnjn0d.js +171 -0
- package/dist/index-3rm7cz6h.js +373 -0
- package/dist/index-3v2z4f0q.js +328 -0
- package/dist/index-48pz4jpc.js +392 -0
- package/dist/index-4j3tt0c3.js +234 -0
- package/dist/index-5n05se68.js +307 -0
- package/dist/index-5vwpzb5a.js +764 -0
- package/dist/index-6g2pv9d2.js +370 -0
- package/dist/index-7fzc32gf.js +712 -0
- package/dist/index-84x0vjjp.js +432 -0
- package/dist/index-adx6khj5.js +488 -0
- package/dist/index-b88v3vtm.js +805 -0
- package/dist/index-cv6fh7z5.js +462 -0
- package/dist/index-cxfrakt7.js +1067 -0
- package/dist/index-d13v9ckt.js +150 -0
- package/dist/index-dycjbhw2.js +1943 -0
- package/dist/index-dyqwejk5.js +531 -0
- package/dist/index-e5fbsywq.js +658 -0
- package/dist/index-f984hw45.js +927 -0
- package/dist/index-gh719d91.js +63 -0
- package/dist/index-hgve9rh2.js +2046 -0
- package/dist/index-hya40gb2.js +474 -0
- package/dist/index-j4zgmzjr.js +152 -0
- package/dist/index-jsmvyyvf.js +1166 -0
- package/dist/index-kh0pwvdd.js +1974 -0
- package/dist/index-mqx4nd6v.js +119 -0
- package/dist/index-mxxxytys.js +1254 -0
- package/dist/index-n5dd7r0v.js +711 -0
- package/dist/index-npg9z1a4.js +431 -0
- package/dist/index-pj501bh1.js +140 -0
- package/dist/index-qry4vhxk.js +1041 -0
- package/dist/index-s2gw5aw9.js +199 -0
- package/dist/index-vvdj6kca.js +70 -0
- package/dist/index-w2zc0vwa.js +571 -0
- package/dist/index-x3fthpsc.js +585 -0
- package/dist/index-zr53sf63.js +44 -0
- package/dist/index-zxdy5pby.js +2728 -0
- package/dist/index-zzhgcwyt.js +966 -0
- package/dist/index.js +453 -0
- package/dist/navigation.js +13 -0
- package/dist/pdf.js +85 -0
- package/dist/percolate.js +24 -0
- package/dist/portfolio.js +77 -0
- package/dist/query.js +29 -0
- package/dist/repository-memory.js +60 -0
- package/dist/sdk.js +36 -0
- package/dist/search-rules.js +34 -0
- package/dist/search.js +22 -0
- package/dist/semantic.js +39 -0
- package/dist/source-inbox.js +17 -0
- package/dist/untrusted-content.js +32 -0
- package/dist/url-intelligence.js +74 -0
- package/dist/workflow.js +23 -0
- package/dist/workflows/decision-context.js +20 -0
- package/dist/workflows/explain-change.js +8 -0
- package/dist/workflows/index.js +28 -0
- package/dist/workflows/plan-radar.js +10 -0
- package/package.json +409 -0
- package/skills/kb/AGENTS.md +19 -0
- package/skills/kb/SKILL.md +82 -0
- package/skills/kb/agents/openai.yaml +4 -0
- package/skills/kb/references/pdf-review.md +51 -0
- package/skills/kb/references/percolate.md +144 -0
- package/skills/kb/references/plan-structure.md +105 -0
- package/skills/kb/references/plan.md +115 -0
- package/skills/kb/references/query.md +231 -0
- package/skills/kb/references/refresh.md +133 -0
- package/skills/kb/references/save-pdf.md +125 -0
- package/skills/kb/references/save-url.md +178 -0
- package/skills/kb/references/url-authentication.md +75 -0
- package/skills/kb/references/url-platforms.md +25 -0
- package/src/agent-context.ts +1241 -0
- package/src/agent-guide-audit.ts +750 -0
- package/src/attachments.ts +723 -0
- package/src/authoring.ts +1725 -0
- package/src/benchmark.ts +362 -0
- package/src/capture.ts +68 -0
- package/src/cli.ts +3660 -0
- package/src/clip/acquire.ts +1273 -0
- package/src/clip/archive-today.ts +402 -0
- package/src/clip/args.ts +423 -0
- package/src/clip/assets.ts +262 -0
- package/src/clip/bounded-byte-buffer.ts +75 -0
- package/src/clip/browser-profiles.ts +372 -0
- package/src/clip/bundle-reader.ts +608 -0
- package/src/clip/capture.ts +1054 -0
- package/src/clip/cli.ts +181 -0
- package/src/clip/cookies.ts +512 -0
- package/src/clip/doctor.ts +1110 -0
- package/src/clip/extract.ts +868 -0
- package/src/clip/jobs.ts +959 -0
- package/src/clip/lib.ts +861 -0
- package/src/clip/media.ts +1474 -0
- package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
- package/src/clip/metadata-search-tool/Cargo.toml +18 -0
- package/src/clip/metadata-search-tool/runner.ts +685 -0
- package/src/clip/metadata-search-tool/src/main.rs +389 -0
- package/src/clip/metadata-search.ts +687 -0
- package/src/clip/network-proxy.ts +627 -0
- package/src/clip/network.ts +950 -0
- package/src/clip/package-root.ts +54 -0
- package/src/clip/persist.ts +1472 -0
- package/src/clip/platforms.ts +1824 -0
- package/src/clip/refresh.ts +197 -0
- package/src/clip/structured.ts +432 -0
- package/src/clip/terminal.ts +197 -0
- package/src/clip/url-intelligence.ts +581 -0
- package/src/clip/url-metadata-backfill.ts +616 -0
- package/src/clip/url-metadata-cli.ts +251 -0
- package/src/clip/url-metadata.ts +1397 -0
- package/src/evaluation-analysis-v2.ts +1440 -0
- package/src/evaluation-builder.ts +2539 -0
- package/src/evaluation-corpus-authoring.ts +2052 -0
- package/src/evaluation-evidence.ts +1147 -0
- package/src/evaluation-execution-v2.ts +426 -0
- package/src/evaluation-implementation.ts +131 -0
- package/src/evaluation-kb-closure.ts +1430 -0
- package/src/evaluation-kb-runner-v2.ts +735 -0
- package/src/evaluation-kb-v2.ts +2567 -0
- package/src/evaluation-kb.ts +1018 -0
- package/src/evaluation-measurement-v2.ts +1462 -0
- package/src/evaluation-packing-v2.ts +452 -0
- package/src/evaluation-redaction.ts +28 -0
- package/src/evaluation-v2.ts +3857 -0
- package/src/evaluation.ts +1220 -0
- package/src/git.ts +1276 -0
- package/src/graph.ts +1487 -0
- package/src/index.ts +20 -0
- package/src/init.ts +123 -0
- package/src/navigation.ts +277 -0
- package/src/note-lock.ts +529 -0
- package/src/pdf/args.ts +202 -0
- package/src/pdf/capture.ts +298 -0
- package/src/pdf/cli.ts +159 -0
- package/src/pdf/extract.ts +303 -0
- package/src/pdf/layout.ts +618 -0
- package/src/pdf/markdown.ts +154 -0
- package/src/pdf/model.ts +234 -0
- package/src/pdf/ocr.ts +190 -0
- package/src/pdf/persist.ts +314 -0
- package/src/pdf/source.ts +135 -0
- package/src/pdf/tools.ts +146 -0
- package/src/pdf.ts +92 -0
- package/src/percolate.ts +877 -0
- package/src/portfolio-audit.ts +574 -0
- package/src/portfolio-identity.ts +162 -0
- package/src/portfolio-registry.ts +473 -0
- package/src/portfolio.ts +765 -0
- package/src/query.ts +646 -0
- package/src/repository-memory.ts +1150 -0
- package/src/sdk.ts +1452 -0
- package/src/search-rules.ts +712 -0
- package/src/search.ts +710 -0
- package/src/semantic-runtime.ts +1139 -0
- package/src/semantic.ts +2480 -0
- package/src/source-inbox.ts +232 -0
- package/src/untrusted-content.ts +401 -0
- package/src/url-intelligence.ts +104 -0
- package/src/vault.ts +636 -0
- package/src/workflow.ts +653 -0
- package/src/workflows/decision-context.ts +81 -0
- package/src/workflows/explain-change.ts +73 -0
- package/src/workflows/index.ts +3 -0
- package/src/workflows/plan-radar.ts +101 -0
|
@@ -0,0 +1,1824 @@
|
|
|
1
|
+
/** URL classification and pure normalization for structured social captures. */
|
|
2
|
+
|
|
3
|
+
import { rewriteContent } from "./lib.js";
|
|
4
|
+
|
|
5
|
+
export type PlatformUrl =
|
|
6
|
+
| { readonly platform: "x"; readonly href: string; readonly handle: string; readonly postId: string }
|
|
7
|
+
| { readonly platform: "hacker-news"; readonly href: string; readonly itemId: string }
|
|
8
|
+
| {
|
|
9
|
+
readonly platform: "reddit";
|
|
10
|
+
readonly href: string;
|
|
11
|
+
readonly postId: string;
|
|
12
|
+
readonly subreddit: string | null;
|
|
13
|
+
readonly commentId: string | null;
|
|
14
|
+
}
|
|
15
|
+
| {
|
|
16
|
+
readonly platform: "bluesky";
|
|
17
|
+
readonly href: string;
|
|
18
|
+
readonly actor: string;
|
|
19
|
+
readonly postId: string;
|
|
20
|
+
}
|
|
21
|
+
| { readonly platform: "substack"; readonly href: string; readonly publication: string | null }
|
|
22
|
+
| { readonly platform: "instagram"; readonly href: string; readonly contentId: string | null }
|
|
23
|
+
| { readonly platform: "linkedin"; readonly href: string; readonly contentId: string | null }
|
|
24
|
+
| { readonly platform: "facebook"; readonly href: string; readonly contentId: string | null }
|
|
25
|
+
| { readonly platform: "tiktok"; readonly href: string; readonly contentId: string | null }
|
|
26
|
+
| { readonly platform: "threads"; readonly href: string; readonly contentId: string | null }
|
|
27
|
+
| { readonly platform: "whatsapp"; readonly href: string; readonly contentId: string | null }
|
|
28
|
+
| { readonly platform: "youtube"; readonly href: string; readonly contentId: string | null }
|
|
29
|
+
| {
|
|
30
|
+
readonly platform: "github";
|
|
31
|
+
readonly href: string;
|
|
32
|
+
readonly owner: string;
|
|
33
|
+
readonly repository: string;
|
|
34
|
+
readonly contentKind: "issue" | "pull-request" | "discussion";
|
|
35
|
+
readonly contentId: string;
|
|
36
|
+
}
|
|
37
|
+
| { readonly platform: "discourse"; readonly href: string; readonly topicId: string }
|
|
38
|
+
| { readonly platform: "generic"; readonly href: string; readonly host: string };
|
|
39
|
+
|
|
40
|
+
export type Platform = PlatformUrl["platform"];
|
|
41
|
+
|
|
42
|
+
export const classifiedPlatforms = [
|
|
43
|
+
"x",
|
|
44
|
+
"hacker-news",
|
|
45
|
+
"reddit",
|
|
46
|
+
"bluesky",
|
|
47
|
+
"substack",
|
|
48
|
+
"instagram",
|
|
49
|
+
"linkedin",
|
|
50
|
+
"facebook",
|
|
51
|
+
"tiktok",
|
|
52
|
+
"threads",
|
|
53
|
+
"whatsapp",
|
|
54
|
+
"youtube",
|
|
55
|
+
"github",
|
|
56
|
+
"discourse",
|
|
57
|
+
"generic",
|
|
58
|
+
] as const satisfies readonly Platform[];
|
|
59
|
+
|
|
60
|
+
export type CapturedPlatform = "x" | "hacker-news" | "reddit" | "bluesky";
|
|
61
|
+
export type CapturedRole = "post" | "comment" | "quote";
|
|
62
|
+
|
|
63
|
+
export type CapturedAuthor = {
|
|
64
|
+
readonly name: string;
|
|
65
|
+
readonly handle: string | null;
|
|
66
|
+
readonly profileUrl: string | null;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
export type CapturedMedia = {
|
|
70
|
+
readonly kind: "image" | "video" | "gif" | "link";
|
|
71
|
+
readonly url: string;
|
|
72
|
+
readonly previewUrl: string | null;
|
|
73
|
+
readonly alt: string | null;
|
|
74
|
+
readonly title: string | null;
|
|
75
|
+
readonly dimensions: { readonly width: number; readonly height: number } | null;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
export type CapturedMetrics = {
|
|
79
|
+
readonly score: number | null;
|
|
80
|
+
readonly replies: number | null;
|
|
81
|
+
readonly likes: number | null;
|
|
82
|
+
readonly reposts: number | null;
|
|
83
|
+
readonly quotes: number | null;
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
export type CapturedContentEntry = {
|
|
87
|
+
readonly kind: "content";
|
|
88
|
+
readonly role: CapturedRole;
|
|
89
|
+
readonly id: string;
|
|
90
|
+
readonly author: CapturedAuthor | null;
|
|
91
|
+
readonly createdAt: string | null;
|
|
92
|
+
readonly sourceUrl: string | null;
|
|
93
|
+
readonly text: string;
|
|
94
|
+
readonly media: readonly CapturedMedia[];
|
|
95
|
+
readonly metrics: CapturedMetrics;
|
|
96
|
+
readonly quotes: readonly CapturedEntry[];
|
|
97
|
+
readonly replies: readonly CapturedEntry[];
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
export type CapturedUnavailableEntry = {
|
|
101
|
+
readonly kind: "unavailable";
|
|
102
|
+
readonly role: CapturedRole;
|
|
103
|
+
readonly id: string;
|
|
104
|
+
readonly reason: "deleted" | "dead" | "removed" | "blocked" | "not-found";
|
|
105
|
+
readonly sourceUrl: string | null;
|
|
106
|
+
readonly replies: readonly CapturedEntry[];
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
export type CapturedMoreEntry = {
|
|
110
|
+
readonly kind: "more";
|
|
111
|
+
readonly id: string;
|
|
112
|
+
readonly count: number | null;
|
|
113
|
+
readonly childIds: readonly string[];
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
export type CapturedBoundaryEntry = {
|
|
117
|
+
readonly kind: "boundary";
|
|
118
|
+
readonly reason: "cycle" | "depth-limit" | "item-limit";
|
|
119
|
+
readonly detail: string;
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
export type CapturedEntry =
|
|
123
|
+
| CapturedContentEntry
|
|
124
|
+
| CapturedUnavailableEntry
|
|
125
|
+
| CapturedMoreEntry
|
|
126
|
+
| CapturedBoundaryEntry;
|
|
127
|
+
|
|
128
|
+
export type NonEmptyEntries = readonly [CapturedEntry, ...CapturedEntry[]];
|
|
129
|
+
|
|
130
|
+
export type CapturedDocument = {
|
|
131
|
+
readonly platform: CapturedPlatform;
|
|
132
|
+
readonly sourceUrl: string;
|
|
133
|
+
readonly title: string;
|
|
134
|
+
readonly ancestors: readonly CapturedEntry[];
|
|
135
|
+
readonly roots: NonEmptyEntries;
|
|
136
|
+
readonly warnings: readonly string[];
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
export type CaptureError = {
|
|
140
|
+
readonly code: "invalid-source" | "invalid-shape" | "missing-root";
|
|
141
|
+
readonly message: string;
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
export type CaptureResult =
|
|
145
|
+
| { readonly ok: true; readonly document: CapturedDocument }
|
|
146
|
+
| { readonly ok: false; readonly error: CaptureError };
|
|
147
|
+
|
|
148
|
+
export type CaptureLimits = {
|
|
149
|
+
readonly maxDepth: number;
|
|
150
|
+
readonly maxItems: number;
|
|
151
|
+
readonly maxTextLength: number;
|
|
152
|
+
readonly maxMediaPerEntry: number;
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
export type CaptureOptions = {
|
|
156
|
+
readonly limits?: Partial<CaptureLimits>;
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
export const DEFAULT_CAPTURE_LIMITS: CaptureLimits = {
|
|
160
|
+
maxDepth: 24,
|
|
161
|
+
maxItems: 1_000,
|
|
162
|
+
maxTextLength: 100_000,
|
|
163
|
+
maxMediaPerEntry: 32,
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const HARD_CAPTURE_LIMITS: CaptureLimits = {
|
|
167
|
+
maxDepth: 64,
|
|
168
|
+
maxItems: 10_000,
|
|
169
|
+
maxTextLength: 1_000_000,
|
|
170
|
+
maxMediaPerEntry: 128,
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
type ParseContext = {
|
|
174
|
+
readonly limits: CaptureLimits;
|
|
175
|
+
usedItems: number;
|
|
176
|
+
readonly warnings: string[];
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
const isUnknownArray = (value: unknown): value is unknown[] => Array.isArray(value);
|
|
180
|
+
|
|
181
|
+
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
182
|
+
typeof value === "object" && value !== null && !isUnknownArray(value);
|
|
183
|
+
|
|
184
|
+
const nonEmptyString = (value: unknown): string | null =>
|
|
185
|
+
typeof value === "string" && value.trim() !== "" ? value.trim() : null;
|
|
186
|
+
|
|
187
|
+
const stringValue = (value: unknown): string | null => (typeof value === "string" ? value : null);
|
|
188
|
+
|
|
189
|
+
const booleanValue = (value: unknown): boolean | null =>
|
|
190
|
+
typeof value === "boolean" ? value : null;
|
|
191
|
+
|
|
192
|
+
const safeInteger = (value: unknown): number | null =>
|
|
193
|
+
typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
194
|
+
|
|
195
|
+
const signedSafeInteger = (value: unknown): number | null =>
|
|
196
|
+
typeof value === "number" && Number.isSafeInteger(value) ? value : null;
|
|
197
|
+
|
|
198
|
+
const finiteNumber = (value: unknown): number | null =>
|
|
199
|
+
typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
|
|
200
|
+
|
|
201
|
+
const foreignId = (value: unknown): string | null => {
|
|
202
|
+
const text = nonEmptyString(value);
|
|
203
|
+
if (text !== null) return text;
|
|
204
|
+
const number = safeInteger(value);
|
|
205
|
+
return number === null ? null : String(number);
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
const readRecord = (record: Record<string, unknown>, key: string): Record<string, unknown> | null => {
|
|
209
|
+
const value = record[key];
|
|
210
|
+
return isRecord(value) ? value : null;
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
const readArray = (record: Record<string, unknown>, key: string): readonly unknown[] | null => {
|
|
214
|
+
const value = record[key];
|
|
215
|
+
return isUnknownArray(value) ? value : null;
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
const clampLimit = (value: number | undefined, fallback: number, ceiling: number): number => {
|
|
219
|
+
if (value === undefined || !Number.isFinite(value)) return fallback;
|
|
220
|
+
return Math.max(1, Math.min(Math.floor(value), ceiling));
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
const createContext = (options: CaptureOptions | undefined): ParseContext => ({
|
|
224
|
+
limits: {
|
|
225
|
+
maxDepth: clampLimit(
|
|
226
|
+
options?.limits?.maxDepth,
|
|
227
|
+
DEFAULT_CAPTURE_LIMITS.maxDepth,
|
|
228
|
+
HARD_CAPTURE_LIMITS.maxDepth,
|
|
229
|
+
),
|
|
230
|
+
maxItems: clampLimit(
|
|
231
|
+
options?.limits?.maxItems,
|
|
232
|
+
DEFAULT_CAPTURE_LIMITS.maxItems,
|
|
233
|
+
HARD_CAPTURE_LIMITS.maxItems,
|
|
234
|
+
),
|
|
235
|
+
maxTextLength: clampLimit(
|
|
236
|
+
options?.limits?.maxTextLength,
|
|
237
|
+
DEFAULT_CAPTURE_LIMITS.maxTextLength,
|
|
238
|
+
HARD_CAPTURE_LIMITS.maxTextLength,
|
|
239
|
+
),
|
|
240
|
+
maxMediaPerEntry: clampLimit(
|
|
241
|
+
options?.limits?.maxMediaPerEntry,
|
|
242
|
+
DEFAULT_CAPTURE_LIMITS.maxMediaPerEntry,
|
|
243
|
+
HARD_CAPTURE_LIMITS.maxMediaPerEntry,
|
|
244
|
+
),
|
|
245
|
+
},
|
|
246
|
+
usedItems: 0,
|
|
247
|
+
warnings: [],
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
const warn = (context: ParseContext, message: string): void => {
|
|
251
|
+
if (context.warnings.length < 100 && !context.warnings.includes(message)) {
|
|
252
|
+
context.warnings.push(message);
|
|
253
|
+
}
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
const reserveItem = (context: ParseContext): boolean => {
|
|
257
|
+
if (context.usedItems >= context.limits.maxItems) {
|
|
258
|
+
warn(context, `Capture stopped at ${context.limits.maxItems} items.`);
|
|
259
|
+
return false;
|
|
260
|
+
}
|
|
261
|
+
context.usedItems += 1;
|
|
262
|
+
return true;
|
|
263
|
+
};
|
|
264
|
+
|
|
265
|
+
const boundary = (reason: CapturedBoundaryEntry["reason"], detail: string): CapturedBoundaryEntry => ({
|
|
266
|
+
kind: "boundary",
|
|
267
|
+
reason,
|
|
268
|
+
detail,
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
const emptyMetrics = (): CapturedMetrics => ({
|
|
272
|
+
score: null,
|
|
273
|
+
replies: null,
|
|
274
|
+
likes: null,
|
|
275
|
+
reposts: null,
|
|
276
|
+
quotes: null,
|
|
277
|
+
});
|
|
278
|
+
|
|
279
|
+
const asNonEmpty = (entries: readonly CapturedEntry[]): NonEmptyEntries | null => {
|
|
280
|
+
const [first, ...rest] = entries;
|
|
281
|
+
return first === undefined ? null : [first, ...rest];
|
|
282
|
+
};
|
|
283
|
+
|
|
284
|
+
const httpUrl = (value: unknown, base?: string): string | null => {
|
|
285
|
+
const text = nonEmptyString(value);
|
|
286
|
+
if (text === null) return null;
|
|
287
|
+
try {
|
|
288
|
+
const url = base === undefined ? new URL(text) : new URL(text, base);
|
|
289
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url.href : null;
|
|
290
|
+
} catch {
|
|
291
|
+
return null;
|
|
292
|
+
}
|
|
293
|
+
};
|
|
294
|
+
|
|
295
|
+
const normalizedSource = (sourceUrl: string): string | null => httpUrl(sourceUrl);
|
|
296
|
+
|
|
297
|
+
const boundedText = (value: string, context: ParseContext, label: string): string => {
|
|
298
|
+
if (value.length <= context.limits.maxTextLength) return value;
|
|
299
|
+
warn(context, `${label} was truncated to ${context.limits.maxTextLength} characters.`);
|
|
300
|
+
return `${value.slice(0, context.limits.maxTextLength)}\n\n[Text truncated.]`;
|
|
301
|
+
};
|
|
302
|
+
|
|
303
|
+
const boundedTitle = (value: string, context: ParseContext, label: string): string => {
|
|
304
|
+
const limit = Math.min(context.limits.maxTextLength, 512);
|
|
305
|
+
if (value.length <= limit) return value;
|
|
306
|
+
warn(context, `${label} was truncated to ${limit} characters.`);
|
|
307
|
+
return `${value.slice(0, Math.max(1, limit - 1))}…`;
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
const isoTimestamp = (value: unknown): string | null => {
|
|
311
|
+
const text = nonEmptyString(value);
|
|
312
|
+
if (text === null) return null;
|
|
313
|
+
const milliseconds = Date.parse(text);
|
|
314
|
+
return Number.isFinite(milliseconds) ? new Date(milliseconds).toISOString() : null;
|
|
315
|
+
};
|
|
316
|
+
|
|
317
|
+
const epochTimestamp = (value: unknown): string | null => {
|
|
318
|
+
const seconds = finiteNumber(value);
|
|
319
|
+
if (seconds === null || seconds > 253_402_300_799) return null;
|
|
320
|
+
return new Date(seconds * 1_000).toISOString();
|
|
321
|
+
};
|
|
322
|
+
|
|
323
|
+
const cleanPathSegments = (url: URL): string[] => {
|
|
324
|
+
const segments: string[] = [];
|
|
325
|
+
for (const rawSegment of url.pathname.split("/")) {
|
|
326
|
+
if (rawSegment === "") continue;
|
|
327
|
+
try {
|
|
328
|
+
segments.push(decodeURIComponent(rawSegment));
|
|
329
|
+
} catch {
|
|
330
|
+
segments.push(rawSegment);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
return segments;
|
|
334
|
+
};
|
|
335
|
+
|
|
336
|
+
const domainMatches = (hostname: string, domain: string): boolean =>
|
|
337
|
+
hostname === domain || hostname.endsWith(`.${domain}`);
|
|
338
|
+
|
|
339
|
+
const canonicalWithoutFragment = (url: URL): string => {
|
|
340
|
+
const canonical = new URL(url.href);
|
|
341
|
+
canonical.hash = "";
|
|
342
|
+
return canonical.href;
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
/** Classify a public content URL without accepting lookalike domains or non-web schemes. */
|
|
346
|
+
export function classifyPlatformUrl(value: string | URL): PlatformUrl | null {
|
|
347
|
+
let url: URL;
|
|
348
|
+
try {
|
|
349
|
+
url = new URL(typeof value === "string" ? value : value.href);
|
|
350
|
+
} catch {
|
|
351
|
+
return null;
|
|
352
|
+
}
|
|
353
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
|
354
|
+
|
|
355
|
+
const hostname = url.hostname.toLowerCase();
|
|
356
|
+
const segments = cleanPathSegments(url);
|
|
357
|
+
|
|
358
|
+
if (domainMatches(hostname, "x.com") || domainMatches(hostname, "twitter.com")) {
|
|
359
|
+
const handle = segments[0];
|
|
360
|
+
const status = segments[1];
|
|
361
|
+
const postId = segments[2];
|
|
362
|
+
if (
|
|
363
|
+
handle !== undefined &&
|
|
364
|
+
status === "status" &&
|
|
365
|
+
postId !== undefined &&
|
|
366
|
+
/^[a-zA-Z0-9_]{1,32}$/.test(handle) &&
|
|
367
|
+
/^\d+$/.test(postId)
|
|
368
|
+
) {
|
|
369
|
+
return {
|
|
370
|
+
platform: "x",
|
|
371
|
+
href: `https://x.com/${handle}/status/${postId}`,
|
|
372
|
+
handle,
|
|
373
|
+
postId,
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
if (hostname === "news.ycombinator.com" && url.pathname === "/item") {
|
|
379
|
+
const itemId = url.searchParams.get("id");
|
|
380
|
+
if (itemId !== null && /^\d+$/.test(itemId)) {
|
|
381
|
+
return {
|
|
382
|
+
platform: "hacker-news",
|
|
383
|
+
href: `https://news.ycombinator.com/item?id=${itemId}`,
|
|
384
|
+
itemId,
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
if (domainMatches(hostname, "reddit.com")) {
|
|
390
|
+
const commentsIndex = segments.indexOf("comments");
|
|
391
|
+
const postId = commentsIndex >= 0 ? segments[commentsIndex + 1] : undefined;
|
|
392
|
+
if (postId !== undefined && /^[a-zA-Z0-9]+$/.test(postId)) {
|
|
393
|
+
const subreddit = commentsIndex >= 2 && segments[0] === "r" ? (segments[1] ?? null) : null;
|
|
394
|
+
const possibleComment = segments[commentsIndex + 3];
|
|
395
|
+
const commentId =
|
|
396
|
+
possibleComment !== undefined && /^[a-zA-Z0-9]+$/.test(possibleComment)
|
|
397
|
+
? possibleComment
|
|
398
|
+
: null;
|
|
399
|
+
return {
|
|
400
|
+
platform: "reddit",
|
|
401
|
+
href: canonicalWithoutFragment(url),
|
|
402
|
+
postId,
|
|
403
|
+
subreddit,
|
|
404
|
+
commentId,
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
if (hostname === "redd.it") {
|
|
409
|
+
const postId = segments[0];
|
|
410
|
+
if (postId !== undefined && /^[a-zA-Z0-9]+$/.test(postId)) {
|
|
411
|
+
return {
|
|
412
|
+
platform: "reddit",
|
|
413
|
+
href: canonicalWithoutFragment(url),
|
|
414
|
+
postId,
|
|
415
|
+
subreddit: null,
|
|
416
|
+
commentId: null,
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
if (hostname === "bsky.app" && segments[0] === "profile" && segments[2] === "post") {
|
|
422
|
+
const actor = segments[1];
|
|
423
|
+
const postId = segments[3];
|
|
424
|
+
if (actor !== undefined && actor !== "" && postId !== undefined && postId !== "") {
|
|
425
|
+
return {
|
|
426
|
+
platform: "bluesky",
|
|
427
|
+
href: `https://bsky.app/profile/${encodeURIComponent(actor)}/post/${encodeURIComponent(postId)}`,
|
|
428
|
+
actor,
|
|
429
|
+
postId,
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
if (hostname === "substack.com" || domainMatches(hostname, "substack.com")) {
|
|
435
|
+
const publication = hostname === "substack.com" ? null : hostname.slice(0, -".substack.com".length);
|
|
436
|
+
return { platform: "substack", href: canonicalWithoutFragment(url), publication };
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
if (domainMatches(hostname, "instagram.com")) {
|
|
440
|
+
const contentId = ["p", "reel", "tv"].includes(segments[0] ?? "") ? (segments[1] ?? null) : null;
|
|
441
|
+
return { platform: "instagram", href: canonicalWithoutFragment(url), contentId };
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
if (domainMatches(hostname, "linkedin.com")) {
|
|
445
|
+
const contentId = segments.find((segment) => /(?:activity|ugcPost|share)[:-]?\d+/.test(segment)) ??
|
|
446
|
+
segments[1] ??
|
|
447
|
+
null;
|
|
448
|
+
return { platform: "linkedin", href: canonicalWithoutFragment(url), contentId };
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
if (
|
|
452
|
+
domainMatches(hostname, "facebook.com") ||
|
|
453
|
+
hostname === "fb.com" ||
|
|
454
|
+
hostname === "fb.watch"
|
|
455
|
+
) {
|
|
456
|
+
const contentId = url.searchParams.get("story_fbid") ?? url.searchParams.get("v") ?? segments.at(-1) ?? null;
|
|
457
|
+
return { platform: "facebook", href: canonicalWithoutFragment(url), contentId };
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
if (domainMatches(hostname, "tiktok.com")) {
|
|
461
|
+
const videoIndex = segments.indexOf("video");
|
|
462
|
+
const contentId = videoIndex >= 0 ? (segments[videoIndex + 1] ?? null) : (segments[0] ?? null);
|
|
463
|
+
return { platform: "tiktok", href: canonicalWithoutFragment(url), contentId };
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
if (domainMatches(hostname, "threads.com") || domainMatches(hostname, "threads.net")) {
|
|
467
|
+
const postIndex = segments.indexOf("post");
|
|
468
|
+
const contentId = postIndex >= 0 ? (segments[postIndex + 1] ?? null) : null;
|
|
469
|
+
return { platform: "threads", href: canonicalWithoutFragment(url), contentId };
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
if (hostname === "web.whatsapp.com") {
|
|
473
|
+
return { platform: "whatsapp", href: canonicalWithoutFragment(url), contentId: null };
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
if (domainMatches(hostname, "youtube.com") || hostname === "youtu.be") {
|
|
477
|
+
const contentId = hostname === "youtu.be"
|
|
478
|
+
? (segments[0] ?? null)
|
|
479
|
+
: url.searchParams.get("v") ?? (segments[0] === "shorts" || segments[0] === "live" ? (segments[1] ?? null) : null);
|
|
480
|
+
return { platform: "youtube", href: canonicalWithoutFragment(url), contentId };
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
if (hostname === "github.com") {
|
|
484
|
+
const owner = segments[0];
|
|
485
|
+
const repository = segments[1];
|
|
486
|
+
const route = segments[2];
|
|
487
|
+
const contentId = segments[3];
|
|
488
|
+
const contentKind = route === "issues"
|
|
489
|
+
? "issue"
|
|
490
|
+
: route === "pull"
|
|
491
|
+
? "pull-request"
|
|
492
|
+
: route === "discussions"
|
|
493
|
+
? "discussion"
|
|
494
|
+
: null;
|
|
495
|
+
if (
|
|
496
|
+
owner !== undefined
|
|
497
|
+
&& repository !== undefined
|
|
498
|
+
&& contentKind !== null
|
|
499
|
+
&& contentId !== undefined
|
|
500
|
+
&& /^[A-Za-z0-9_.-]+$/.test(owner)
|
|
501
|
+
&& /^[A-Za-z0-9_.-]+$/.test(repository)
|
|
502
|
+
&& /^\d+$/.test(contentId)
|
|
503
|
+
) {
|
|
504
|
+
return {
|
|
505
|
+
platform: "github",
|
|
506
|
+
href: canonicalWithoutFragment(url),
|
|
507
|
+
owner,
|
|
508
|
+
repository,
|
|
509
|
+
contentKind,
|
|
510
|
+
contentId,
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
const discourseHostHint = hostname
|
|
516
|
+
.split(".")
|
|
517
|
+
.some((label) => label === "discourse" || label === "discuss" || label === "forum" || label === "community");
|
|
518
|
+
if (discourseHostHint && segments[0] === "t") {
|
|
519
|
+
const topicId = /^\d+$/.test(segments[1] ?? "")
|
|
520
|
+
? segments[1]
|
|
521
|
+
: /^\d+$/.test(segments[2] ?? "")
|
|
522
|
+
? segments[2]
|
|
523
|
+
: undefined;
|
|
524
|
+
if (topicId !== undefined) {
|
|
525
|
+
return { platform: "discourse", href: canonicalWithoutFragment(url), topicId };
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
return { platform: "generic", href: canonicalWithoutFragment(url), host: hostname };
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
const invalidSource = (): CaptureResult => ({
|
|
533
|
+
ok: false,
|
|
534
|
+
error: { code: "invalid-source", message: "The capture source must be an HTTP(S) URL." },
|
|
535
|
+
});
|
|
536
|
+
|
|
537
|
+
const invalidShape = (message: string): CaptureResult => ({
|
|
538
|
+
ok: false,
|
|
539
|
+
error: { code: "invalid-shape", message },
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
type BirdItem = {
|
|
543
|
+
readonly id: string;
|
|
544
|
+
readonly parentId: string | null;
|
|
545
|
+
readonly articleTitle: string | null;
|
|
546
|
+
readonly entry: CapturedContentEntry;
|
|
547
|
+
};
|
|
548
|
+
|
|
549
|
+
const birdMedia = (value: unknown, context: ParseContext): CapturedMedia[] => {
|
|
550
|
+
if (!isUnknownArray(value)) return [];
|
|
551
|
+
const media: CapturedMedia[] = [];
|
|
552
|
+
const limit = Math.min(value.length, context.limits.maxMediaPerEntry);
|
|
553
|
+
for (let index = 0; index < limit; index += 1) {
|
|
554
|
+
const item = value[index];
|
|
555
|
+
if (!isRecord(item)) continue;
|
|
556
|
+
const rawType = nonEmptyString(item.type);
|
|
557
|
+
const imageUrl = httpUrl(item.url);
|
|
558
|
+
const videoUrl = httpUrl(item.videoUrl);
|
|
559
|
+
const url = rawType === "video" || rawType === "animated_gif" ? (videoUrl ?? imageUrl) : imageUrl;
|
|
560
|
+
if (url === null) continue;
|
|
561
|
+
const width = safeInteger(item.width);
|
|
562
|
+
const height = safeInteger(item.height);
|
|
563
|
+
media.push({
|
|
564
|
+
kind: rawType === "video" ? "video" : rawType === "animated_gif" ? "gif" : "image",
|
|
565
|
+
url,
|
|
566
|
+
previewUrl: httpUrl(item.previewUrl) ?? (videoUrl === null ? null : imageUrl),
|
|
567
|
+
alt: nonEmptyString(item.altText) ?? nonEmptyString(item.alt),
|
|
568
|
+
title: null,
|
|
569
|
+
dimensions: width === null || height === null ? null : { width, height },
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
if (value.length > limit) warn(context, `Media was truncated to ${limit} items on one entry.`);
|
|
573
|
+
return media;
|
|
574
|
+
};
|
|
575
|
+
|
|
576
|
+
const birdAuthor = (value: unknown): CapturedAuthor | null => {
|
|
577
|
+
if (!isRecord(value)) return null;
|
|
578
|
+
const handle = nonEmptyString(value.username);
|
|
579
|
+
if (handle === null) return null;
|
|
580
|
+
return {
|
|
581
|
+
name: nonEmptyString(value.name) ?? handle,
|
|
582
|
+
handle,
|
|
583
|
+
profileUrl: `https://x.com/${encodeURIComponent(handle)}`,
|
|
584
|
+
};
|
|
585
|
+
};
|
|
586
|
+
|
|
587
|
+
function parseBirdItem(
|
|
588
|
+
value: unknown,
|
|
589
|
+
context: ParseContext,
|
|
590
|
+
depth: number,
|
|
591
|
+
active: WeakSet<object>,
|
|
592
|
+
): BirdItem | null {
|
|
593
|
+
if (!isRecord(value)) return null;
|
|
594
|
+
const id = foreignId(value.id);
|
|
595
|
+
const text = stringValue(value.text);
|
|
596
|
+
const author = birdAuthor(value.author);
|
|
597
|
+
if (id === null || text === null || author === null) return null;
|
|
598
|
+
if (active.has(value)) return null;
|
|
599
|
+
if (!reserveItem(context)) return null;
|
|
600
|
+
active.add(value);
|
|
601
|
+
|
|
602
|
+
const quotes: CapturedEntry[] = [];
|
|
603
|
+
const quotedValue = value.quotedTweet;
|
|
604
|
+
if (quotedValue !== undefined && quotedValue !== null) {
|
|
605
|
+
if (depth >= context.limits.maxDepth) {
|
|
606
|
+
quotes.push(boundary("depth-limit", `Quoted post nesting exceeded ${context.limits.maxDepth}.`));
|
|
607
|
+
warn(context, `Quoted post nesting stopped at depth ${context.limits.maxDepth}.`);
|
|
608
|
+
} else if (isRecord(quotedValue) && active.has(quotedValue)) {
|
|
609
|
+
quotes.push(boundary("cycle", `Quoted post ${id} references an ancestor.`));
|
|
610
|
+
warn(context, "A cycle in quoted posts was stopped.");
|
|
611
|
+
} else {
|
|
612
|
+
const quoted = parseBirdItem(quotedValue, context, depth + 1, active);
|
|
613
|
+
if (quoted === null) warn(context, `Malformed quoted post on ${id} was skipped.`);
|
|
614
|
+
else quotes.push({ ...quoted.entry, role: "quote" });
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
const article = readRecord(value, "article");
|
|
619
|
+
const articleTitle = article === null ? null : nonEmptyString(article.title);
|
|
620
|
+
const entry: CapturedContentEntry = {
|
|
621
|
+
kind: "content",
|
|
622
|
+
role: "post",
|
|
623
|
+
id,
|
|
624
|
+
author,
|
|
625
|
+
createdAt: isoTimestamp(value.createdAt),
|
|
626
|
+
sourceUrl: `https://x.com/${encodeURIComponent(author.handle ?? "")}/status/${encodeURIComponent(id)}`,
|
|
627
|
+
text: boundedText(text, context, `Post ${id}`),
|
|
628
|
+
media: birdMedia(value.media, context),
|
|
629
|
+
metrics: {
|
|
630
|
+
score: null,
|
|
631
|
+
replies: safeInteger(value.replyCount),
|
|
632
|
+
likes: safeInteger(value.likeCount),
|
|
633
|
+
reposts: safeInteger(value.retweetCount),
|
|
634
|
+
quotes: safeInteger(value.quoteCount),
|
|
635
|
+
},
|
|
636
|
+
quotes,
|
|
637
|
+
replies: [],
|
|
638
|
+
};
|
|
639
|
+
active.delete(value);
|
|
640
|
+
return {
|
|
641
|
+
id,
|
|
642
|
+
parentId: foreignId(value.inReplyToStatusId),
|
|
643
|
+
articleTitle,
|
|
644
|
+
entry,
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
const birdValues = (input: unknown): readonly unknown[] | null => {
|
|
649
|
+
if (isUnknownArray(input)) return input;
|
|
650
|
+
if (!isRecord(input)) return null;
|
|
651
|
+
const tweets = readArray(input, "tweets");
|
|
652
|
+
if (tweets !== null) return tweets;
|
|
653
|
+
if (input.tweet !== undefined) return [input.tweet];
|
|
654
|
+
return [input];
|
|
655
|
+
};
|
|
656
|
+
|
|
657
|
+
const firstLine = (text: string, maxLength: number): string => {
|
|
658
|
+
const line = text.split(/\r?\n/, 1)[0]?.trim() ?? "";
|
|
659
|
+
return line.length <= maxLength ? line : `${line.slice(0, Math.max(1, maxLength - 1))}…`;
|
|
660
|
+
};
|
|
661
|
+
|
|
662
|
+
/** Parse `bird read/thread --json` output into the shared capture tree. */
|
|
663
|
+
export function parseBirdCapture(
|
|
664
|
+
input: unknown,
|
|
665
|
+
sourceUrl: string,
|
|
666
|
+
options?: CaptureOptions,
|
|
667
|
+
): CaptureResult {
|
|
668
|
+
const source = normalizedSource(sourceUrl);
|
|
669
|
+
if (source === null) return invalidSource();
|
|
670
|
+
const values = birdValues(input);
|
|
671
|
+
if (values === null) return invalidShape("Bird output must be a tweet, tweet array, or tweets wrapper.");
|
|
672
|
+
|
|
673
|
+
const context = createContext(options);
|
|
674
|
+
const items = new Map<string, BirdItem>();
|
|
675
|
+
const order: string[] = [];
|
|
676
|
+
const scanLimit = Math.min(values.length, context.limits.maxItems);
|
|
677
|
+
for (let index = 0; index < scanLimit; index += 1) {
|
|
678
|
+
const item = parseBirdItem(values[index], context, 0, new WeakSet<object>());
|
|
679
|
+
if (item === null) {
|
|
680
|
+
warn(context, `Malformed Bird item at index ${index} was skipped.`);
|
|
681
|
+
continue;
|
|
682
|
+
}
|
|
683
|
+
if (items.has(item.id)) {
|
|
684
|
+
warn(context, `Duplicate Bird post ${item.id} was skipped.`);
|
|
685
|
+
continue;
|
|
686
|
+
}
|
|
687
|
+
items.set(item.id, item);
|
|
688
|
+
order.push(item.id);
|
|
689
|
+
}
|
|
690
|
+
if (values.length > scanLimit) warn(context, `Bird output was truncated to ${scanLimit} top-level items.`);
|
|
691
|
+
if (order.length === 0) return invalidShape("Bird output contained no valid tweets.");
|
|
692
|
+
|
|
693
|
+
const children = new Map<string, string[]>();
|
|
694
|
+
for (const id of order) {
|
|
695
|
+
const item = items.get(id);
|
|
696
|
+
if (item?.parentId === null || item?.parentId === undefined || !items.has(item.parentId)) continue;
|
|
697
|
+
const existing = children.get(item.parentId);
|
|
698
|
+
if (existing === undefined) children.set(item.parentId, [id]);
|
|
699
|
+
else existing.push(id);
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
const rendered = new Set<string>();
|
|
703
|
+
const buildTree = (id: string, path: ReadonlySet<string>, depth: number): CapturedEntry => {
|
|
704
|
+
const item = items.get(id);
|
|
705
|
+
if (item === undefined) return boundary("cycle", `Post ${id} could not be resolved.`);
|
|
706
|
+
if (depth >= context.limits.maxDepth) {
|
|
707
|
+
return boundary("depth-limit", `Thread nesting exceeded ${context.limits.maxDepth}.`);
|
|
708
|
+
}
|
|
709
|
+
if (path.has(id)) {
|
|
710
|
+
warn(context, "A cycle in Bird reply relationships was stopped.");
|
|
711
|
+
return boundary("cycle", `Post ${id} repeats in its reply ancestry.`);
|
|
712
|
+
}
|
|
713
|
+
rendered.add(id);
|
|
714
|
+
const nextPath = new Set(path);
|
|
715
|
+
nextPath.add(id);
|
|
716
|
+
const replies = (children.get(id) ?? []).map((childId) => buildTree(childId, nextPath, depth + 1));
|
|
717
|
+
return { ...item.entry, replies };
|
|
718
|
+
};
|
|
719
|
+
|
|
720
|
+
const classified = classifyPlatformUrl(source);
|
|
721
|
+
const targetId = classified?.platform === "x" ? classified.postId : null;
|
|
722
|
+
let targetRoot = targetId;
|
|
723
|
+
const walked = new Set<string>();
|
|
724
|
+
for (let depth = 0; targetRoot !== null && depth < context.limits.maxDepth; depth += 1) {
|
|
725
|
+
if (walked.has(targetRoot)) break;
|
|
726
|
+
walked.add(targetRoot);
|
|
727
|
+
const parentId = items.get(targetRoot)?.parentId ?? null;
|
|
728
|
+
if (parentId === null || !items.has(parentId)) break;
|
|
729
|
+
targetRoot = parentId;
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
const rootIds: string[] = [];
|
|
733
|
+
if (targetRoot !== null && items.has(targetRoot)) rootIds.push(targetRoot);
|
|
734
|
+
for (const id of order) {
|
|
735
|
+
const parentId = items.get(id)?.parentId ?? null;
|
|
736
|
+
if ((parentId === null || !items.has(parentId)) && !rootIds.includes(id)) rootIds.push(id);
|
|
737
|
+
}
|
|
738
|
+
const roots: CapturedEntry[] = rootIds.map((id) => buildTree(id, new Set<string>(), 0));
|
|
739
|
+
for (const id of order) {
|
|
740
|
+
if (!rendered.has(id)) roots.push(buildTree(id, new Set<string>(), 0));
|
|
741
|
+
}
|
|
742
|
+
const nonEmptyRoots = asNonEmpty(roots);
|
|
743
|
+
if (nonEmptyRoots === null) {
|
|
744
|
+
return { ok: false, error: { code: "missing-root", message: "Bird output had no renderable root." } };
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
const titleItem = (targetId === null ? undefined : items.get(targetId)) ?? items.get(order[0] ?? "");
|
|
748
|
+
const authorLabel = titleItem?.entry.author?.handle ?? titleItem?.entry.author?.name ?? "Unknown author";
|
|
749
|
+
const rawTitle =
|
|
750
|
+
titleItem?.articleTitle ?? (firstLine(titleItem?.entry.text ?? "", 96) || `@${authorLabel} on X`);
|
|
751
|
+
const title = boundedTitle(rawTitle, context, "Bird title");
|
|
752
|
+
return {
|
|
753
|
+
ok: true,
|
|
754
|
+
document: {
|
|
755
|
+
platform: "x",
|
|
756
|
+
sourceUrl: source,
|
|
757
|
+
title,
|
|
758
|
+
ancestors: [],
|
|
759
|
+
roots: nonEmptyRoots,
|
|
760
|
+
warnings: context.warnings,
|
|
761
|
+
},
|
|
762
|
+
};
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
const decodeHtmlEntities = (value: string): string =>
|
|
766
|
+
value.replace(/&(?:#(\d+)|#x([0-9a-f]+)|([a-z]+));/gi, (whole, decimal, hexadecimal, named) => {
|
|
767
|
+
if (typeof decimal === "string") {
|
|
768
|
+
const point = Number.parseInt(decimal, 10);
|
|
769
|
+
return Number.isSafeInteger(point) && point <= 0x10ffff ? String.fromCodePoint(point) : whole;
|
|
770
|
+
}
|
|
771
|
+
if (typeof hexadecimal === "string") {
|
|
772
|
+
const point = Number.parseInt(hexadecimal, 16);
|
|
773
|
+
return Number.isSafeInteger(point) && point <= 0x10ffff ? String.fromCodePoint(point) : whole;
|
|
774
|
+
}
|
|
775
|
+
if (typeof named !== "string") return whole;
|
|
776
|
+
const entities: Readonly<Record<string, string>> = {
|
|
777
|
+
amp: "&",
|
|
778
|
+
apos: "'",
|
|
779
|
+
gt: ">",
|
|
780
|
+
lt: "<",
|
|
781
|
+
nbsp: " ",
|
|
782
|
+
quot: '"',
|
|
783
|
+
};
|
|
784
|
+
return entities[named.toLowerCase()] ?? whole;
|
|
785
|
+
});
|
|
786
|
+
|
|
787
|
+
type HtmlTag = {
|
|
788
|
+
readonly closing: boolean;
|
|
789
|
+
readonly name: string;
|
|
790
|
+
readonly raw: string;
|
|
791
|
+
};
|
|
792
|
+
|
|
793
|
+
const parseHtmlTag = (raw: string): HtmlTag | null => {
|
|
794
|
+
let cursor = 0;
|
|
795
|
+
let closing = false;
|
|
796
|
+
if (raw.charCodeAt(cursor) === 47) {
|
|
797
|
+
closing = true;
|
|
798
|
+
cursor += 1;
|
|
799
|
+
}
|
|
800
|
+
const start = cursor;
|
|
801
|
+
while (cursor < raw.length) {
|
|
802
|
+
const code = raw.charCodeAt(cursor);
|
|
803
|
+
const alphaNumeric =
|
|
804
|
+
(code >= 48 && code <= 57) ||
|
|
805
|
+
(code >= 65 && code <= 90) ||
|
|
806
|
+
(code >= 97 && code <= 122);
|
|
807
|
+
if (!alphaNumeric) break;
|
|
808
|
+
cursor += 1;
|
|
809
|
+
}
|
|
810
|
+
if (cursor === start) return null;
|
|
811
|
+
return { closing, name: raw.slice(start, cursor).toLowerCase(), raw };
|
|
812
|
+
};
|
|
813
|
+
|
|
814
|
+
const stripHtmlTagsLinear = (html: string): string => {
|
|
815
|
+
const chunks: string[] = [];
|
|
816
|
+
let cursor = 0;
|
|
817
|
+
while (cursor < html.length) {
|
|
818
|
+
const opening = html.indexOf("<", cursor);
|
|
819
|
+
if (opening < 0) {
|
|
820
|
+
chunks.push(html.slice(cursor));
|
|
821
|
+
break;
|
|
822
|
+
}
|
|
823
|
+
if (opening > cursor) chunks.push(html.slice(cursor, opening));
|
|
824
|
+
const closing = html.indexOf(">", opening + 1);
|
|
825
|
+
if (closing < 0) {
|
|
826
|
+
chunks.push(html.slice(opening));
|
|
827
|
+
break;
|
|
828
|
+
}
|
|
829
|
+
cursor = closing + 1;
|
|
830
|
+
}
|
|
831
|
+
return chunks.join("");
|
|
832
|
+
};
|
|
833
|
+
|
|
834
|
+
const quotedHref = (tag: HtmlTag): string | null => {
|
|
835
|
+
let cursor = tag.name.length;
|
|
836
|
+
while (cursor < tag.raw.length) {
|
|
837
|
+
while (cursor < tag.raw.length && /\s/.test(tag.raw[cursor] ?? "")) cursor += 1;
|
|
838
|
+
if (tag.raw[cursor] === "/") {
|
|
839
|
+
cursor += 1;
|
|
840
|
+
continue;
|
|
841
|
+
}
|
|
842
|
+
const nameStart = cursor;
|
|
843
|
+
while (cursor < tag.raw.length) {
|
|
844
|
+
const character = tag.raw[cursor] ?? "";
|
|
845
|
+
if (/\s/.test(character) || character === "=" || character === "/") break;
|
|
846
|
+
cursor += 1;
|
|
847
|
+
}
|
|
848
|
+
if (cursor === nameStart) {
|
|
849
|
+
cursor += 1;
|
|
850
|
+
continue;
|
|
851
|
+
}
|
|
852
|
+
const name = tag.raw.slice(nameStart, cursor).toLowerCase();
|
|
853
|
+
while (cursor < tag.raw.length && /\s/.test(tag.raw[cursor] ?? "")) cursor += 1;
|
|
854
|
+
if (tag.raw[cursor] !== "=") continue;
|
|
855
|
+
cursor += 1;
|
|
856
|
+
while (cursor < tag.raw.length && /\s/.test(tag.raw[cursor] ?? "")) cursor += 1;
|
|
857
|
+
const quote = tag.raw[cursor];
|
|
858
|
+
if (quote !== '"' && quote !== "'") continue;
|
|
859
|
+
cursor += 1;
|
|
860
|
+
const valueStart = cursor;
|
|
861
|
+
while (cursor < tag.raw.length && tag.raw[cursor] !== quote) cursor += 1;
|
|
862
|
+
if (cursor >= tag.raw.length) return null;
|
|
863
|
+
if (name === "href") return tag.raw.slice(valueStart, cursor);
|
|
864
|
+
cursor += 1;
|
|
865
|
+
}
|
|
866
|
+
return null;
|
|
867
|
+
};
|
|
868
|
+
|
|
869
|
+
const isWhitespace = (value: string): boolean => /\s/.test(value);
|
|
870
|
+
|
|
871
|
+
const isPlainBreakTag = (tag: HtmlTag): boolean => {
|
|
872
|
+
if (tag.closing || tag.name !== "br") return false;
|
|
873
|
+
let cursor = tag.name.length;
|
|
874
|
+
while (cursor < tag.raw.length && isWhitespace(tag.raw[cursor] ?? "")) cursor += 1;
|
|
875
|
+
if (tag.raw[cursor] === "/") cursor += 1;
|
|
876
|
+
while (cursor < tag.raw.length && isWhitespace(tag.raw[cursor] ?? "")) cursor += 1;
|
|
877
|
+
return cursor === tag.raw.length;
|
|
878
|
+
};
|
|
879
|
+
|
|
880
|
+
const isExactFormattingTag = (tag: HtmlTag, name: string): boolean =>
|
|
881
|
+
tag.name === name && tag.raw.length === name.length + (tag.closing ? 1 : 0);
|
|
882
|
+
|
|
883
|
+
const htmlToMarkdown = (html: string): string => {
|
|
884
|
+
const chunks: string[] = [];
|
|
885
|
+
const lower = html.toLowerCase();
|
|
886
|
+
let nextAnchorClosing = lower.indexOf("</a>");
|
|
887
|
+
let cursor = 0;
|
|
888
|
+
while (cursor < html.length) {
|
|
889
|
+
const opening = html.indexOf("<", cursor);
|
|
890
|
+
if (opening < 0) {
|
|
891
|
+
chunks.push(html.slice(cursor));
|
|
892
|
+
break;
|
|
893
|
+
}
|
|
894
|
+
if (opening > cursor) chunks.push(html.slice(cursor, opening));
|
|
895
|
+
const closing = html.indexOf(">", opening + 1);
|
|
896
|
+
if (closing < 0) {
|
|
897
|
+
chunks.push(html.slice(opening));
|
|
898
|
+
break;
|
|
899
|
+
}
|
|
900
|
+
const tag = parseHtmlTag(html.slice(opening + 1, closing));
|
|
901
|
+
if (tag === null) {
|
|
902
|
+
cursor = closing + 1;
|
|
903
|
+
continue;
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
if (!tag.closing && tag.name === "a") {
|
|
907
|
+
const target = quotedHref(tag);
|
|
908
|
+
while (nextAnchorClosing >= 0 && nextAnchorClosing < closing + 1) {
|
|
909
|
+
nextAnchorClosing = lower.indexOf("</a>", nextAnchorClosing + 4);
|
|
910
|
+
}
|
|
911
|
+
if (target !== null && nextAnchorClosing >= 0) {
|
|
912
|
+
const label = html.slice(closing + 1, nextAnchorClosing);
|
|
913
|
+
const cleanLabel = decodeHtmlEntities(stripHtmlTagsLinear(label)).trim();
|
|
914
|
+
const url = httpUrl(decodeHtmlEntities(target), "https://news.ycombinator.com/");
|
|
915
|
+
chunks.push(url === null ? cleanLabel : `[${cleanLabel || url}](<${url}>)`);
|
|
916
|
+
cursor = nextAnchorClosing + 4;
|
|
917
|
+
nextAnchorClosing = lower.indexOf("</a>", cursor);
|
|
918
|
+
continue;
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
const nextOpening = closing + 1;
|
|
923
|
+
if (
|
|
924
|
+
isExactFormattingTag(tag, "pre") &&
|
|
925
|
+
!tag.closing &&
|
|
926
|
+
lower.startsWith("<code>", nextOpening)
|
|
927
|
+
) {
|
|
928
|
+
chunks.push("\n\n```\n");
|
|
929
|
+
cursor = nextOpening + 6;
|
|
930
|
+
continue;
|
|
931
|
+
}
|
|
932
|
+
if (
|
|
933
|
+
isExactFormattingTag(tag, "code") &&
|
|
934
|
+
tag.closing &&
|
|
935
|
+
lower.startsWith("</pre>", nextOpening)
|
|
936
|
+
) {
|
|
937
|
+
chunks.push("\n```\n\n");
|
|
938
|
+
cursor = nextOpening + 6;
|
|
939
|
+
continue;
|
|
940
|
+
}
|
|
941
|
+
if (!tag.closing && tag.name === "p") chunks.push("\n\n");
|
|
942
|
+
else if (isPlainBreakTag(tag)) chunks.push("\n");
|
|
943
|
+
else if (isExactFormattingTag(tag, "i") || isExactFormattingTag(tag, "em")) chunks.push("*");
|
|
944
|
+
else if (isExactFormattingTag(tag, "b") || isExactFormattingTag(tag, "strong")) chunks.push("**");
|
|
945
|
+
cursor = closing + 1;
|
|
946
|
+
}
|
|
947
|
+
return decodeHtmlEntities(chunks.join(""))
|
|
948
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
949
|
+
.trim();
|
|
950
|
+
};
|
|
951
|
+
|
|
952
|
+
type HackerNewsEnvelope = { readonly root: unknown; readonly descendants: readonly unknown[] };
|
|
953
|
+
|
|
954
|
+
const hackerNewsEnvelope = (input: unknown): HackerNewsEnvelope | null => {
|
|
955
|
+
if (isUnknownArray(input)) {
|
|
956
|
+
const [root, ...descendants] = input;
|
|
957
|
+
return root === undefined ? null : { root, descendants };
|
|
958
|
+
}
|
|
959
|
+
if (!isRecord(input)) return null;
|
|
960
|
+
const root = input.root ?? input.rootItem;
|
|
961
|
+
const descendants = readArray(input, "descendants") ?? readArray(input, "items") ?? [];
|
|
962
|
+
return root === undefined ? null : { root, descendants };
|
|
963
|
+
};
|
|
964
|
+
|
|
965
|
+
const hackerNewsKids = (record: Record<string, unknown>, context: ParseContext): string[] => {
|
|
966
|
+
const kids = readArray(record, "kids");
|
|
967
|
+
if (kids === null) return [];
|
|
968
|
+
const result: string[] = [];
|
|
969
|
+
const limit = Math.min(kids.length, context.limits.maxItems);
|
|
970
|
+
for (let index = 0; index < limit; index += 1) {
|
|
971
|
+
const id = foreignId(kids[index]);
|
|
972
|
+
if (id !== null) result.push(id);
|
|
973
|
+
}
|
|
974
|
+
if (kids.length > limit) warn(context, `Hacker News child IDs were truncated to ${limit}.`);
|
|
975
|
+
return result;
|
|
976
|
+
};
|
|
977
|
+
|
|
978
|
+
/** Parse a Hacker News Firebase root item plus its fetched descendants. */
|
|
979
|
+
export function parseHackerNewsCapture(
|
|
980
|
+
input: unknown,
|
|
981
|
+
sourceUrl: string,
|
|
982
|
+
options?: CaptureOptions,
|
|
983
|
+
): CaptureResult {
|
|
984
|
+
const source = normalizedSource(sourceUrl);
|
|
985
|
+
if (source === null) return invalidSource();
|
|
986
|
+
const envelope = hackerNewsEnvelope(input);
|
|
987
|
+
if (envelope === null || !isRecord(envelope.root)) {
|
|
988
|
+
return invalidShape("Hacker News input must provide a root item and descendant items.");
|
|
989
|
+
}
|
|
990
|
+
const rootId = foreignId(envelope.root.id);
|
|
991
|
+
if (rootId === null) return invalidShape("The Hacker News root item has no valid id.");
|
|
992
|
+
|
|
993
|
+
const context = createContext(options);
|
|
994
|
+
const byId = new Map<string, Record<string, unknown>>();
|
|
995
|
+
byId.set(rootId, envelope.root);
|
|
996
|
+
const scanLimit = Math.min(envelope.descendants.length, context.limits.maxItems - 1);
|
|
997
|
+
for (let index = 0; index < scanLimit; index += 1) {
|
|
998
|
+
const value = envelope.descendants[index];
|
|
999
|
+
if (!isRecord(value)) {
|
|
1000
|
+
warn(context, `Malformed Hacker News descendant at index ${index} was skipped.`);
|
|
1001
|
+
continue;
|
|
1002
|
+
}
|
|
1003
|
+
const id = foreignId(value.id);
|
|
1004
|
+
if (id === null) {
|
|
1005
|
+
warn(context, `Hacker News descendant at index ${index} has no id.`);
|
|
1006
|
+
continue;
|
|
1007
|
+
}
|
|
1008
|
+
if (byId.has(id)) warn(context, `Duplicate Hacker News item ${id} was skipped.`);
|
|
1009
|
+
else byId.set(id, value);
|
|
1010
|
+
}
|
|
1011
|
+
if (envelope.descendants.length > scanLimit) {
|
|
1012
|
+
warn(context, `Hacker News descendants were truncated to ${scanLimit}.`);
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
const buildItem = (
|
|
1016
|
+
id: string,
|
|
1017
|
+
role: CapturedRole,
|
|
1018
|
+
path: ReadonlySet<string>,
|
|
1019
|
+
depth: number,
|
|
1020
|
+
): CapturedEntry => {
|
|
1021
|
+
if (depth >= context.limits.maxDepth) {
|
|
1022
|
+
return boundary("depth-limit", `Hacker News nesting exceeded ${context.limits.maxDepth}.`);
|
|
1023
|
+
}
|
|
1024
|
+
if (path.has(id)) {
|
|
1025
|
+
warn(context, "A cycle in Hacker News child IDs was stopped.");
|
|
1026
|
+
return boundary("cycle", `Hacker News item ${id} repeats in its ancestry.`);
|
|
1027
|
+
}
|
|
1028
|
+
if (!reserveItem(context)) return boundary("item-limit", "The Hacker News item limit was reached.");
|
|
1029
|
+
const record = byId.get(id);
|
|
1030
|
+
if (record === undefined) {
|
|
1031
|
+
return {
|
|
1032
|
+
kind: "unavailable",
|
|
1033
|
+
role,
|
|
1034
|
+
id,
|
|
1035
|
+
reason: "not-found",
|
|
1036
|
+
sourceUrl: `https://news.ycombinator.com/item?id=${encodeURIComponent(id)}`,
|
|
1037
|
+
replies: [],
|
|
1038
|
+
};
|
|
1039
|
+
}
|
|
1040
|
+
const nextPath = new Set(path);
|
|
1041
|
+
nextPath.add(id);
|
|
1042
|
+
const replies: CapturedEntry[] = [];
|
|
1043
|
+
for (const childId of hackerNewsKids(record, context)) {
|
|
1044
|
+
if (context.usedItems >= context.limits.maxItems) {
|
|
1045
|
+
replies.push(boundary("item-limit", "Additional Hacker News descendants were omitted."));
|
|
1046
|
+
break;
|
|
1047
|
+
}
|
|
1048
|
+
replies.push(buildItem(childId, "comment", nextPath, depth + 1));
|
|
1049
|
+
}
|
|
1050
|
+
const sourceForItem = `https://news.ycombinator.com/item?id=${encodeURIComponent(id)}`;
|
|
1051
|
+
if (booleanValue(record.deleted) === true || booleanValue(record.dead) === true) {
|
|
1052
|
+
return {
|
|
1053
|
+
kind: "unavailable",
|
|
1054
|
+
role,
|
|
1055
|
+
id,
|
|
1056
|
+
reason: booleanValue(record.deleted) === true ? "deleted" : "dead",
|
|
1057
|
+
sourceUrl: sourceForItem,
|
|
1058
|
+
replies,
|
|
1059
|
+
};
|
|
1060
|
+
}
|
|
1061
|
+
const authorHandle = nonEmptyString(record.by);
|
|
1062
|
+
const author: CapturedAuthor | null =
|
|
1063
|
+
authorHandle === null
|
|
1064
|
+
? null
|
|
1065
|
+
: {
|
|
1066
|
+
name: authorHandle,
|
|
1067
|
+
handle: authorHandle,
|
|
1068
|
+
profileUrl: `https://news.ycombinator.com/user?id=${encodeURIComponent(authorHandle)}`,
|
|
1069
|
+
};
|
|
1070
|
+
const rawText = stringValue(record.text) ?? "";
|
|
1071
|
+
const body = htmlToMarkdown(boundedText(rawText, context, `Hacker News item ${id}`));
|
|
1072
|
+
const external = httpUrl(record.url, "https://news.ycombinator.com/");
|
|
1073
|
+
const text = external === null ? body : `${body}${body === "" ? "" : "\n\n"}[Linked article](<${external}>)`;
|
|
1074
|
+
return {
|
|
1075
|
+
kind: "content",
|
|
1076
|
+
role,
|
|
1077
|
+
id,
|
|
1078
|
+
author,
|
|
1079
|
+
createdAt: epochTimestamp(record.time),
|
|
1080
|
+
sourceUrl: sourceForItem,
|
|
1081
|
+
text,
|
|
1082
|
+
media: [],
|
|
1083
|
+
metrics: {
|
|
1084
|
+
...emptyMetrics(),
|
|
1085
|
+
score: safeInteger(record.score),
|
|
1086
|
+
replies: safeInteger(record.descendants) ?? (replies.length === 0 ? null : replies.length),
|
|
1087
|
+
},
|
|
1088
|
+
quotes: [],
|
|
1089
|
+
replies,
|
|
1090
|
+
};
|
|
1091
|
+
};
|
|
1092
|
+
|
|
1093
|
+
const root = buildItem(rootId, "post", new Set<string>(), 0);
|
|
1094
|
+
const title = boundedTitle(
|
|
1095
|
+
nonEmptyString(envelope.root.title) ?? `Hacker News item ${rootId}`,
|
|
1096
|
+
context,
|
|
1097
|
+
"Hacker News title",
|
|
1098
|
+
);
|
|
1099
|
+
return {
|
|
1100
|
+
ok: true,
|
|
1101
|
+
document: {
|
|
1102
|
+
platform: "hacker-news",
|
|
1103
|
+
sourceUrl: source,
|
|
1104
|
+
title,
|
|
1105
|
+
ancestors: [],
|
|
1106
|
+
roots: [root],
|
|
1107
|
+
warnings: context.warnings,
|
|
1108
|
+
},
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
const redditListingChildren = (value: unknown): readonly unknown[] | null => {
|
|
1113
|
+
if (!isRecord(value)) return null;
|
|
1114
|
+
if (value.kind === "Listing") {
|
|
1115
|
+
const data = readRecord(value, "data");
|
|
1116
|
+
return data === null ? null : readArray(data, "children");
|
|
1117
|
+
}
|
|
1118
|
+
const data = readRecord(value, "data");
|
|
1119
|
+
return data === null ? null : readArray(data, "children");
|
|
1120
|
+
};
|
|
1121
|
+
|
|
1122
|
+
type RedditEnvelope = { readonly post: unknown; readonly comments: unknown };
|
|
1123
|
+
|
|
1124
|
+
const redditEnvelope = (input: unknown): RedditEnvelope | null => {
|
|
1125
|
+
if (isUnknownArray(input)) {
|
|
1126
|
+
const post = input[0];
|
|
1127
|
+
if (post === undefined) return null;
|
|
1128
|
+
return { post, comments: input[1] ?? null };
|
|
1129
|
+
}
|
|
1130
|
+
if (!isRecord(input) || input.post === undefined) return null;
|
|
1131
|
+
return { post: input.post, comments: input.comments ?? null };
|
|
1132
|
+
};
|
|
1133
|
+
|
|
1134
|
+
const redditPostData = (value: unknown, maxItems: number): Record<string, unknown> | null => {
|
|
1135
|
+
if (isRecord(value) && value.kind === "t3") return readRecord(value, "data");
|
|
1136
|
+
const children = redditListingChildren(value);
|
|
1137
|
+
if (children === null) return null;
|
|
1138
|
+
const limit = Math.min(children.length, maxItems);
|
|
1139
|
+
for (let index = 0; index < limit; index += 1) {
|
|
1140
|
+
const child = children[index];
|
|
1141
|
+
if (isRecord(child) && child.kind === "t3") return readRecord(child, "data");
|
|
1142
|
+
}
|
|
1143
|
+
return null;
|
|
1144
|
+
};
|
|
1145
|
+
|
|
1146
|
+
const redditAuthor = (value: unknown): CapturedAuthor | null => {
|
|
1147
|
+
const handle = nonEmptyString(value);
|
|
1148
|
+
if (handle === null || handle === "[deleted]") return null;
|
|
1149
|
+
return {
|
|
1150
|
+
name: handle,
|
|
1151
|
+
handle,
|
|
1152
|
+
profileUrl: `https://www.reddit.com/user/${encodeURIComponent(handle)}`,
|
|
1153
|
+
};
|
|
1154
|
+
};
|
|
1155
|
+
|
|
1156
|
+
const redditPermalink = (value: unknown): string | null =>
|
|
1157
|
+
httpUrl(value, "https://www.reddit.com/");
|
|
1158
|
+
|
|
1159
|
+
/** Parse Reddit's post-and-comments listing JSON, including nested `more` placeholders. */
|
|
1160
|
+
export function parseRedditCapture(
|
|
1161
|
+
input: unknown,
|
|
1162
|
+
sourceUrl: string,
|
|
1163
|
+
options?: CaptureOptions,
|
|
1164
|
+
): CaptureResult {
|
|
1165
|
+
const source = normalizedSource(sourceUrl);
|
|
1166
|
+
if (source === null) return invalidSource();
|
|
1167
|
+
const envelope = redditEnvelope(input);
|
|
1168
|
+
if (envelope === null) return invalidShape("Reddit input must contain a post listing.");
|
|
1169
|
+
const context = createContext(options);
|
|
1170
|
+
const post = redditPostData(envelope.post, context.limits.maxItems);
|
|
1171
|
+
if (post === null) return invalidShape("Reddit input contained no valid post object.");
|
|
1172
|
+
const postId = foreignId(post.id) ?? foreignId(post.name);
|
|
1173
|
+
const rawTitle = nonEmptyString(post.title);
|
|
1174
|
+
if (postId === null || rawTitle === null) return invalidShape("The Reddit post has no valid id or title.");
|
|
1175
|
+
const title = boundedTitle(rawTitle, context, "Reddit title");
|
|
1176
|
+
if (!reserveItem(context)) return invalidShape("The capture item limit cannot hold the Reddit post.");
|
|
1177
|
+
|
|
1178
|
+
const active = new WeakSet<object>();
|
|
1179
|
+
const parseThing = (value: unknown, depth: number): CapturedEntry | null => {
|
|
1180
|
+
if (!isRecord(value)) return null;
|
|
1181
|
+
if (depth >= context.limits.maxDepth) {
|
|
1182
|
+
return boundary("depth-limit", `Reddit nesting exceeded ${context.limits.maxDepth}.`);
|
|
1183
|
+
}
|
|
1184
|
+
if (active.has(value)) {
|
|
1185
|
+
warn(context, "A cycle in Reddit replies was stopped.");
|
|
1186
|
+
return boundary("cycle", "A Reddit reply object repeats in its ancestry.");
|
|
1187
|
+
}
|
|
1188
|
+
const kind = nonEmptyString(value.kind);
|
|
1189
|
+
const data = readRecord(value, "data");
|
|
1190
|
+
if (kind === null || data === null) return null;
|
|
1191
|
+
if (!reserveItem(context)) return boundary("item-limit", "Additional Reddit comments were omitted.");
|
|
1192
|
+
active.add(value);
|
|
1193
|
+
|
|
1194
|
+
if (kind === "more") {
|
|
1195
|
+
const childValues = readArray(data, "children") ?? [];
|
|
1196
|
+
const childIds: string[] = [];
|
|
1197
|
+
const limit = Math.min(childValues.length, context.limits.maxItems);
|
|
1198
|
+
for (let index = 0; index < limit; index += 1) {
|
|
1199
|
+
const id = foreignId(childValues[index]);
|
|
1200
|
+
if (id !== null) childIds.push(id);
|
|
1201
|
+
}
|
|
1202
|
+
active.delete(value);
|
|
1203
|
+
return {
|
|
1204
|
+
kind: "more",
|
|
1205
|
+
id: foreignId(data.id) ?? "more",
|
|
1206
|
+
count: safeInteger(data.count),
|
|
1207
|
+
childIds,
|
|
1208
|
+
};
|
|
1209
|
+
}
|
|
1210
|
+
if (kind !== "t1") {
|
|
1211
|
+
active.delete(value);
|
|
1212
|
+
warn(context, `Unsupported Reddit thing kind ${kind} was skipped.`);
|
|
1213
|
+
return null;
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
const id = foreignId(data.id) ?? foreignId(data.name) ?? "unknown-comment";
|
|
1217
|
+
const replyValues = redditListingChildren(data.replies) ?? [];
|
|
1218
|
+
const replies: CapturedEntry[] = [];
|
|
1219
|
+
const limit = Math.min(replyValues.length, context.limits.maxItems);
|
|
1220
|
+
for (let index = 0; index < limit; index += 1) {
|
|
1221
|
+
if (context.usedItems >= context.limits.maxItems) {
|
|
1222
|
+
warn(context, `Capture stopped at ${context.limits.maxItems} items.`);
|
|
1223
|
+
replies.push(boundary("item-limit", "Additional Reddit replies were omitted."));
|
|
1224
|
+
break;
|
|
1225
|
+
}
|
|
1226
|
+
const reply = parseThing(replyValues[index], depth + 1);
|
|
1227
|
+
if (reply !== null) replies.push(reply);
|
|
1228
|
+
}
|
|
1229
|
+
const itemSource = redditPermalink(data.permalink);
|
|
1230
|
+
const body = stringValue(data.body) ?? "";
|
|
1231
|
+
active.delete(value);
|
|
1232
|
+
if (body.trim() === "[deleted]" || body.trim() === "[removed]") {
|
|
1233
|
+
return {
|
|
1234
|
+
kind: "unavailable",
|
|
1235
|
+
role: "comment",
|
|
1236
|
+
id,
|
|
1237
|
+
reason: body.trim() === "[deleted]" ? "deleted" : "removed",
|
|
1238
|
+
sourceUrl: itemSource,
|
|
1239
|
+
replies,
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
return {
|
|
1243
|
+
kind: "content",
|
|
1244
|
+
role: "comment",
|
|
1245
|
+
id,
|
|
1246
|
+
author: redditAuthor(data.author),
|
|
1247
|
+
createdAt: epochTimestamp(data.created_utc),
|
|
1248
|
+
sourceUrl: itemSource,
|
|
1249
|
+
text: boundedText(body, context, `Reddit comment ${id}`),
|
|
1250
|
+
media: [],
|
|
1251
|
+
metrics: { ...emptyMetrics(), score: signedSafeInteger(data.score), replies: replies.length || null },
|
|
1252
|
+
quotes: [],
|
|
1253
|
+
replies,
|
|
1254
|
+
};
|
|
1255
|
+
};
|
|
1256
|
+
|
|
1257
|
+
const commentValues = redditListingChildren(envelope.comments) ?? [];
|
|
1258
|
+
const replies: CapturedEntry[] = [];
|
|
1259
|
+
const commentLimit = Math.min(commentValues.length, context.limits.maxItems);
|
|
1260
|
+
for (let index = 0; index < commentLimit; index += 1) {
|
|
1261
|
+
if (context.usedItems >= context.limits.maxItems) {
|
|
1262
|
+
warn(context, `Capture stopped at ${context.limits.maxItems} items.`);
|
|
1263
|
+
replies.push(boundary("item-limit", "Additional Reddit comments were omitted."));
|
|
1264
|
+
break;
|
|
1265
|
+
}
|
|
1266
|
+
const reply = parseThing(commentValues[index], 1);
|
|
1267
|
+
if (reply !== null) replies.push(reply);
|
|
1268
|
+
}
|
|
1269
|
+
if (commentValues.length > commentLimit) warn(context, `Reddit comments were truncated to ${commentLimit}.`);
|
|
1270
|
+
|
|
1271
|
+
const permalink = redditPermalink(post.permalink);
|
|
1272
|
+
const selfText = stringValue(post.selftext) ?? "";
|
|
1273
|
+
const linkedUrl = httpUrl(post.url);
|
|
1274
|
+
const linkText = linkedUrl === null || linkedUrl === permalink ? "" : `[Linked page](<${linkedUrl}>)`;
|
|
1275
|
+
const body = `${boundedText(selfText, context, `Reddit post ${postId}`)}${
|
|
1276
|
+
selfText.trim() === "" || linkText === "" ? "" : "\n\n"
|
|
1277
|
+
}${linkText}`;
|
|
1278
|
+
const root: CapturedContentEntry = {
|
|
1279
|
+
kind: "content",
|
|
1280
|
+
role: "post",
|
|
1281
|
+
id: postId,
|
|
1282
|
+
author: redditAuthor(post.author),
|
|
1283
|
+
createdAt: epochTimestamp(post.created_utc),
|
|
1284
|
+
sourceUrl: permalink ?? source,
|
|
1285
|
+
text: body,
|
|
1286
|
+
media: [],
|
|
1287
|
+
metrics: {
|
|
1288
|
+
...emptyMetrics(),
|
|
1289
|
+
score: signedSafeInteger(post.score),
|
|
1290
|
+
replies: safeInteger(post.num_comments) ?? replies.length,
|
|
1291
|
+
},
|
|
1292
|
+
quotes: [],
|
|
1293
|
+
replies,
|
|
1294
|
+
};
|
|
1295
|
+
return {
|
|
1296
|
+
ok: true,
|
|
1297
|
+
document: {
|
|
1298
|
+
platform: "reddit",
|
|
1299
|
+
sourceUrl: source,
|
|
1300
|
+
title,
|
|
1301
|
+
ancestors: [],
|
|
1302
|
+
roots: [root],
|
|
1303
|
+
warnings: context.warnings,
|
|
1304
|
+
},
|
|
1305
|
+
};
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
const bskyAuthor = (value: unknown): CapturedAuthor | null => {
|
|
1309
|
+
if (!isRecord(value)) return null;
|
|
1310
|
+
const handle = nonEmptyString(value.handle);
|
|
1311
|
+
const did = nonEmptyString(value.did);
|
|
1312
|
+
if (handle === null && did === null) return null;
|
|
1313
|
+
const actor = handle ?? did ?? "unknown";
|
|
1314
|
+
return {
|
|
1315
|
+
name: nonEmptyString(value.displayName) ?? actor,
|
|
1316
|
+
handle,
|
|
1317
|
+
profileUrl: `https://bsky.app/profile/${encodeURIComponent(actor)}`,
|
|
1318
|
+
};
|
|
1319
|
+
};
|
|
1320
|
+
|
|
1321
|
+
const bskyRkey = (uri: string): string | null => {
|
|
1322
|
+
const segments = uri.split("/").filter((segment) => segment !== "");
|
|
1323
|
+
return segments.at(-1) ?? null;
|
|
1324
|
+
};
|
|
1325
|
+
|
|
1326
|
+
const bskyPostUrl = (uri: string | null, author: CapturedAuthor | null): string | null => {
|
|
1327
|
+
if (uri === null || author === null) return null;
|
|
1328
|
+
const rkey = bskyRkey(uri);
|
|
1329
|
+
const actor = author.handle;
|
|
1330
|
+
return rkey === null || actor === null
|
|
1331
|
+
? null
|
|
1332
|
+
: `https://bsky.app/profile/${encodeURIComponent(actor)}/post/${encodeURIComponent(rkey)}`;
|
|
1333
|
+
};
|
|
1334
|
+
|
|
1335
|
+
const bskyMedia = (values: readonly unknown[], context: ParseContext): CapturedMedia[] => {
|
|
1336
|
+
const media: CapturedMedia[] = [];
|
|
1337
|
+
const seen = new Set<string>();
|
|
1338
|
+
const active = new WeakSet<object>();
|
|
1339
|
+
const add = (item: CapturedMedia): void => {
|
|
1340
|
+
const key = `${item.kind}:${item.url}`;
|
|
1341
|
+
if (seen.has(key) || media.length >= context.limits.maxMediaPerEntry) return;
|
|
1342
|
+
seen.add(key);
|
|
1343
|
+
media.push(item);
|
|
1344
|
+
};
|
|
1345
|
+
const visit = (value: unknown, depth: number): void => {
|
|
1346
|
+
if (!isRecord(value) || depth > Math.min(8, context.limits.maxDepth) || active.has(value)) return;
|
|
1347
|
+
active.add(value);
|
|
1348
|
+
const images = readArray(value, "images");
|
|
1349
|
+
if (images !== null) {
|
|
1350
|
+
const limit = Math.min(images.length, context.limits.maxMediaPerEntry);
|
|
1351
|
+
for (let index = 0; index < limit; index += 1) {
|
|
1352
|
+
const image = images[index];
|
|
1353
|
+
if (!isRecord(image)) continue;
|
|
1354
|
+
const url = httpUrl(image.fullsize) ?? httpUrl(image.thumb);
|
|
1355
|
+
if (url === null) continue;
|
|
1356
|
+
const ratio = readRecord(image, "aspectRatio");
|
|
1357
|
+
const width = ratio === null ? null : safeInteger(ratio.width);
|
|
1358
|
+
const height = ratio === null ? null : safeInteger(ratio.height);
|
|
1359
|
+
add({
|
|
1360
|
+
kind: "image",
|
|
1361
|
+
url,
|
|
1362
|
+
previewUrl: httpUrl(image.thumb),
|
|
1363
|
+
alt: nonEmptyString(image.alt),
|
|
1364
|
+
title: null,
|
|
1365
|
+
dimensions: width === null || height === null ? null : { width, height },
|
|
1366
|
+
});
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
const playlist = httpUrl(value.playlist);
|
|
1370
|
+
if (playlist !== null) {
|
|
1371
|
+
add({
|
|
1372
|
+
kind: "video",
|
|
1373
|
+
url: playlist,
|
|
1374
|
+
previewUrl: httpUrl(value.thumbnail),
|
|
1375
|
+
alt: nonEmptyString(value.alt),
|
|
1376
|
+
title: null,
|
|
1377
|
+
dimensions: null,
|
|
1378
|
+
});
|
|
1379
|
+
}
|
|
1380
|
+
const external = readRecord(value, "external");
|
|
1381
|
+
if (external !== null) {
|
|
1382
|
+
const url = httpUrl(external.uri);
|
|
1383
|
+
if (url !== null) {
|
|
1384
|
+
add({
|
|
1385
|
+
kind: "link",
|
|
1386
|
+
url,
|
|
1387
|
+
previewUrl: httpUrl(external.thumb),
|
|
1388
|
+
alt: nonEmptyString(external.description),
|
|
1389
|
+
title: nonEmptyString(external.title),
|
|
1390
|
+
dimensions: null,
|
|
1391
|
+
});
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
if (value.media !== undefined) visit(value.media, depth + 1);
|
|
1395
|
+
const embeds = readArray(value, "embeds");
|
|
1396
|
+
if (embeds !== null) {
|
|
1397
|
+
const limit = Math.min(embeds.length, context.limits.maxMediaPerEntry);
|
|
1398
|
+
for (let index = 0; index < limit; index += 1) visit(embeds[index], depth + 1);
|
|
1399
|
+
}
|
|
1400
|
+
active.delete(value);
|
|
1401
|
+
};
|
|
1402
|
+
for (const value of values) visit(value, 0);
|
|
1403
|
+
if (media.length >= context.limits.maxMediaPerEntry) {
|
|
1404
|
+
warn(context, `Bluesky media was truncated to ${context.limits.maxMediaPerEntry} items on one entry.`);
|
|
1405
|
+
}
|
|
1406
|
+
return media;
|
|
1407
|
+
};
|
|
1408
|
+
|
|
1409
|
+
const bskyQuoteRecord = (embed: unknown): unknown => {
|
|
1410
|
+
if (!isRecord(embed)) return null;
|
|
1411
|
+
const type = nonEmptyString(embed.$type) ?? "";
|
|
1412
|
+
if (type.includes("recordWithMedia")) {
|
|
1413
|
+
const outerRecord = readRecord(embed, "record");
|
|
1414
|
+
return outerRecord?.record ?? null;
|
|
1415
|
+
}
|
|
1416
|
+
if (type.includes("record#view") || type.includes("recordWithMedia#view")) return embed.record ?? null;
|
|
1417
|
+
return null;
|
|
1418
|
+
};
|
|
1419
|
+
|
|
1420
|
+
function parseBskyQuote(
|
|
1421
|
+
value: unknown,
|
|
1422
|
+
context: ParseContext,
|
|
1423
|
+
depth: number,
|
|
1424
|
+
active: WeakSet<object>,
|
|
1425
|
+
): CapturedEntry | null {
|
|
1426
|
+
if (!isRecord(value)) return null;
|
|
1427
|
+
if (depth >= context.limits.maxDepth) {
|
|
1428
|
+
return boundary("depth-limit", `Bluesky quote nesting exceeded ${context.limits.maxDepth}.`);
|
|
1429
|
+
}
|
|
1430
|
+
if (active.has(value)) {
|
|
1431
|
+
warn(context, "A cycle in Bluesky quoted records was stopped.");
|
|
1432
|
+
return boundary("cycle", "A Bluesky quoted record repeats in its ancestry.");
|
|
1433
|
+
}
|
|
1434
|
+
const uri = nonEmptyString(value.uri) ?? "unknown-quote";
|
|
1435
|
+
if (booleanValue(value.notFound) === true || booleanValue(value.blocked) === true) {
|
|
1436
|
+
if (!reserveItem(context)) return boundary("item-limit", "Additional Bluesky quotes were omitted.");
|
|
1437
|
+
return {
|
|
1438
|
+
kind: "unavailable",
|
|
1439
|
+
role: "quote",
|
|
1440
|
+
id: uri,
|
|
1441
|
+
reason: booleanValue(value.blocked) === true ? "blocked" : "not-found",
|
|
1442
|
+
sourceUrl: null,
|
|
1443
|
+
replies: [],
|
|
1444
|
+
};
|
|
1445
|
+
}
|
|
1446
|
+
const author = bskyAuthor(value.author);
|
|
1447
|
+
const record = readRecord(value, "value") ?? readRecord(value, "record");
|
|
1448
|
+
if (record === null || !reserveItem(context)) return null;
|
|
1449
|
+
active.add(value);
|
|
1450
|
+
const nestedEmbed = value.embeds ?? record.embed;
|
|
1451
|
+
const nestedQuoteValue = bskyQuoteRecord(nestedEmbed);
|
|
1452
|
+
const quotes: CapturedEntry[] = [];
|
|
1453
|
+
if (nestedQuoteValue !== null) {
|
|
1454
|
+
const nested = parseBskyQuote(nestedQuoteValue, context, depth + 1, active);
|
|
1455
|
+
if (nested !== null) quotes.push(nested);
|
|
1456
|
+
}
|
|
1457
|
+
active.delete(value);
|
|
1458
|
+
return {
|
|
1459
|
+
kind: "content",
|
|
1460
|
+
role: "quote",
|
|
1461
|
+
id: uri,
|
|
1462
|
+
author,
|
|
1463
|
+
createdAt: isoTimestamp(record.createdAt),
|
|
1464
|
+
sourceUrl: bskyPostUrl(uri, author),
|
|
1465
|
+
text: boundedText(stringValue(record.text) ?? "", context, `Bluesky quote ${uri}`),
|
|
1466
|
+
media: bskyMedia([value, record], context),
|
|
1467
|
+
metrics: emptyMetrics(),
|
|
1468
|
+
quotes,
|
|
1469
|
+
replies: [],
|
|
1470
|
+
};
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
function parseBskyThreadNode(
|
|
1474
|
+
value: unknown,
|
|
1475
|
+
role: CapturedRole,
|
|
1476
|
+
includeReplies: boolean,
|
|
1477
|
+
context: ParseContext,
|
|
1478
|
+
depth: number,
|
|
1479
|
+
active: WeakSet<object>,
|
|
1480
|
+
): CapturedEntry | null {
|
|
1481
|
+
if (!isRecord(value)) return null;
|
|
1482
|
+
if (depth >= context.limits.maxDepth) {
|
|
1483
|
+
return boundary("depth-limit", `Bluesky nesting exceeded ${context.limits.maxDepth}.`);
|
|
1484
|
+
}
|
|
1485
|
+
if (active.has(value)) {
|
|
1486
|
+
warn(context, "A cycle in Bluesky thread objects was stopped.");
|
|
1487
|
+
return boundary("cycle", "A Bluesky thread object repeats in its ancestry.");
|
|
1488
|
+
}
|
|
1489
|
+
const post = readRecord(value, "post");
|
|
1490
|
+
const fallbackUri = nonEmptyString(value.uri) ?? "unknown-post";
|
|
1491
|
+
const type = nonEmptyString(value.$type) ?? "";
|
|
1492
|
+
if (
|
|
1493
|
+
booleanValue(value.notFound) === true ||
|
|
1494
|
+
booleanValue(value.blocked) === true ||
|
|
1495
|
+
type.includes("notFoundPost") ||
|
|
1496
|
+
type.includes("blockedPost")
|
|
1497
|
+
) {
|
|
1498
|
+
if (!reserveItem(context)) return boundary("item-limit", "Additional Bluesky entries were omitted.");
|
|
1499
|
+
return {
|
|
1500
|
+
kind: "unavailable",
|
|
1501
|
+
role,
|
|
1502
|
+
id: fallbackUri,
|
|
1503
|
+
reason: booleanValue(value.blocked) === true || type.includes("blockedPost") ? "blocked" : "not-found",
|
|
1504
|
+
sourceUrl: null,
|
|
1505
|
+
replies: [],
|
|
1506
|
+
};
|
|
1507
|
+
}
|
|
1508
|
+
if (post === null || !reserveItem(context)) return null;
|
|
1509
|
+
const uri = nonEmptyString(post.uri) ?? fallbackUri;
|
|
1510
|
+
const author = bskyAuthor(post.author);
|
|
1511
|
+
const record = readRecord(post, "record");
|
|
1512
|
+
if (record === null) return null;
|
|
1513
|
+
active.add(value);
|
|
1514
|
+
|
|
1515
|
+
const embedValues: unknown[] = [];
|
|
1516
|
+
if (post.embed !== undefined) embedValues.push(post.embed);
|
|
1517
|
+
if (record.embed !== undefined) embedValues.push(record.embed);
|
|
1518
|
+
const quotes: CapturedEntry[] = [];
|
|
1519
|
+
for (const embed of embedValues) {
|
|
1520
|
+
const quoteValue = bskyQuoteRecord(embed);
|
|
1521
|
+
if (quoteValue === null) continue;
|
|
1522
|
+
const quote = parseBskyQuote(quoteValue, context, depth + 1, active);
|
|
1523
|
+
if (quote !== null) quotes.push(quote);
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
const replies: CapturedEntry[] = [];
|
|
1527
|
+
const replyValues = includeReplies ? readArray(value, "replies") ?? [] : [];
|
|
1528
|
+
const limit = Math.min(replyValues.length, context.limits.maxItems);
|
|
1529
|
+
for (let index = 0; index < limit; index += 1) {
|
|
1530
|
+
if (context.usedItems >= context.limits.maxItems) {
|
|
1531
|
+
warn(context, `Capture stopped at ${context.limits.maxItems} items.`);
|
|
1532
|
+
replies.push(boundary("item-limit", "Additional Bluesky replies were omitted."));
|
|
1533
|
+
break;
|
|
1534
|
+
}
|
|
1535
|
+
const reply = parseBskyThreadNode(replyValues[index], "comment", true, context, depth + 1, active);
|
|
1536
|
+
if (reply !== null) replies.push(reply);
|
|
1537
|
+
}
|
|
1538
|
+
active.delete(value);
|
|
1539
|
+
return {
|
|
1540
|
+
kind: "content",
|
|
1541
|
+
role,
|
|
1542
|
+
id: uri,
|
|
1543
|
+
author,
|
|
1544
|
+
createdAt: isoTimestamp(record.createdAt) ?? isoTimestamp(post.indexedAt),
|
|
1545
|
+
sourceUrl: bskyPostUrl(uri, author),
|
|
1546
|
+
text: boundedText(stringValue(record.text) ?? "", context, `Bluesky post ${uri}`),
|
|
1547
|
+
media: bskyMedia(embedValues, context),
|
|
1548
|
+
metrics: {
|
|
1549
|
+
score: null,
|
|
1550
|
+
replies: safeInteger(post.replyCount),
|
|
1551
|
+
likes: safeInteger(post.likeCount),
|
|
1552
|
+
reposts: safeInteger(post.repostCount),
|
|
1553
|
+
quotes: safeInteger(post.quoteCount),
|
|
1554
|
+
},
|
|
1555
|
+
quotes,
|
|
1556
|
+
replies,
|
|
1557
|
+
};
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
/** Parse `app.bsky.feed.getPostThread` JSON, retaining parents, replies, and embeds. */
|
|
1561
|
+
export function parseBlueskyCapture(
|
|
1562
|
+
input: unknown,
|
|
1563
|
+
sourceUrl: string,
|
|
1564
|
+
options?: CaptureOptions,
|
|
1565
|
+
): CaptureResult {
|
|
1566
|
+
const source = normalizedSource(sourceUrl);
|
|
1567
|
+
if (source === null) return invalidSource();
|
|
1568
|
+
if (!isRecord(input)) return invalidShape("Bluesky output must be an object containing a thread.");
|
|
1569
|
+
const thread = input.thread ?? input;
|
|
1570
|
+
if (!isRecord(thread)) return invalidShape("Bluesky output contained no thread root.");
|
|
1571
|
+
const context = createContext(options);
|
|
1572
|
+
const root = parseBskyThreadNode(thread, "post", true, context, 0, new WeakSet<object>());
|
|
1573
|
+
if (root === null) return invalidShape("Bluesky output contained no valid post at the thread root.");
|
|
1574
|
+
|
|
1575
|
+
const ancestorsNearestFirst: CapturedEntry[] = [];
|
|
1576
|
+
const seenParents = new WeakSet<object>();
|
|
1577
|
+
let parent: unknown = thread.parent;
|
|
1578
|
+
for (let depth = 1; parent !== undefined && parent !== null; depth += 1) {
|
|
1579
|
+
if (depth >= context.limits.maxDepth) {
|
|
1580
|
+
ancestorsNearestFirst.push(boundary("depth-limit", "Additional Bluesky parent context was omitted."));
|
|
1581
|
+
break;
|
|
1582
|
+
}
|
|
1583
|
+
if (!isRecord(parent)) break;
|
|
1584
|
+
if (seenParents.has(parent)) {
|
|
1585
|
+
ancestorsNearestFirst.push(boundary("cycle", "A Bluesky parent object repeats in its ancestry."));
|
|
1586
|
+
warn(context, "A cycle in Bluesky parent context was stopped.");
|
|
1587
|
+
break;
|
|
1588
|
+
}
|
|
1589
|
+
seenParents.add(parent);
|
|
1590
|
+
const parsed = parseBskyThreadNode(parent, "post", false, context, depth, new WeakSet<object>());
|
|
1591
|
+
if (parsed !== null) ancestorsNearestFirst.push(parsed);
|
|
1592
|
+
parent = parent.parent;
|
|
1593
|
+
}
|
|
1594
|
+
ancestorsNearestFirst.reverse();
|
|
1595
|
+
const titleAuthor = root.kind === "content" ? root.author : null;
|
|
1596
|
+
const titleText = root.kind === "content" ? firstLine(root.text, 96) : "";
|
|
1597
|
+
const title = titleText || `${titleAuthor?.name ?? "Unknown author"} on Bluesky`;
|
|
1598
|
+
return {
|
|
1599
|
+
ok: true,
|
|
1600
|
+
document: {
|
|
1601
|
+
platform: "bluesky",
|
|
1602
|
+
sourceUrl: source,
|
|
1603
|
+
title,
|
|
1604
|
+
ancestors: ancestorsNearestFirst,
|
|
1605
|
+
roots: [root],
|
|
1606
|
+
warnings: context.warnings,
|
|
1607
|
+
},
|
|
1608
|
+
};
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
/** Dispatch one of the four structured platform parsers. */
|
|
1612
|
+
export function parseStructuredCapture(
|
|
1613
|
+
platform: CapturedPlatform,
|
|
1614
|
+
input: unknown,
|
|
1615
|
+
sourceUrl: string,
|
|
1616
|
+
options?: CaptureOptions,
|
|
1617
|
+
): CaptureResult {
|
|
1618
|
+
switch (platform) {
|
|
1619
|
+
case "x":
|
|
1620
|
+
return parseBirdCapture(input, sourceUrl, options);
|
|
1621
|
+
case "hacker-news":
|
|
1622
|
+
return parseHackerNewsCapture(input, sourceUrl, options);
|
|
1623
|
+
case "reddit":
|
|
1624
|
+
return parseRedditCapture(input, sourceUrl, options);
|
|
1625
|
+
case "bluesky":
|
|
1626
|
+
return parseBlueskyCapture(input, sourceUrl, options);
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1630
|
+
const platformLabel = (platform: CapturedPlatform): string => {
|
|
1631
|
+
switch (platform) {
|
|
1632
|
+
case "x":
|
|
1633
|
+
return "X";
|
|
1634
|
+
case "hacker-news":
|
|
1635
|
+
return "Hacker News";
|
|
1636
|
+
case "reddit":
|
|
1637
|
+
return "Reddit";
|
|
1638
|
+
case "bluesky":
|
|
1639
|
+
return "Bluesky";
|
|
1640
|
+
}
|
|
1641
|
+
};
|
|
1642
|
+
|
|
1643
|
+
const escapeInline = (value: string): string =>
|
|
1644
|
+
value.replace(/\\/g, "\\\\").replace(/([*_[\]`])/g, "\\$1").replace(/\s+/g, " ").trim();
|
|
1645
|
+
|
|
1646
|
+
const cleanHeading = (value: string): string => value.replace(/[\r\n]+/g, " ").replace(/\s+/g, " ").trim();
|
|
1647
|
+
|
|
1648
|
+
const authorLabel = (author: CapturedAuthor | null): string => {
|
|
1649
|
+
if (author === null) return "Unknown author";
|
|
1650
|
+
const name = escapeInline(author.name);
|
|
1651
|
+
if (author.handle === null || author.handle === author.name) return name;
|
|
1652
|
+
return `${name} (@${escapeInline(author.handle)})`;
|
|
1653
|
+
};
|
|
1654
|
+
|
|
1655
|
+
const metadata = (entry: CapturedContentEntry): string => {
|
|
1656
|
+
const pieces = [`**${authorLabel(entry.author)}**`];
|
|
1657
|
+
if (entry.createdAt !== null) pieces.push(entry.createdAt);
|
|
1658
|
+
if (entry.sourceUrl !== null) pieces.push(`[source](<${entry.sourceUrl}>)`);
|
|
1659
|
+
return pieces.join(" · ");
|
|
1660
|
+
};
|
|
1661
|
+
|
|
1662
|
+
const metricLabel = (value: number, singular: string, plural = `${singular}s`): string =>
|
|
1663
|
+
`${value} ${value === 1 ? singular : plural}`;
|
|
1664
|
+
|
|
1665
|
+
const metricsLine = (metrics: CapturedMetrics): string | null => {
|
|
1666
|
+
const pieces: string[] = [];
|
|
1667
|
+
if (metrics.score !== null) pieces.push(metricLabel(metrics.score, "point"));
|
|
1668
|
+
if (metrics.replies !== null) pieces.push(metricLabel(metrics.replies, "reply", "replies"));
|
|
1669
|
+
if (metrics.likes !== null) pieces.push(metricLabel(metrics.likes, "like"));
|
|
1670
|
+
if (metrics.reposts !== null) pieces.push(metricLabel(metrics.reposts, "repost"));
|
|
1671
|
+
if (metrics.quotes !== null) pieces.push(metricLabel(metrics.quotes, "quote"));
|
|
1672
|
+
return pieces.length === 0 ? null : pieces.join(" · ");
|
|
1673
|
+
};
|
|
1674
|
+
|
|
1675
|
+
const indentLines = (lines: readonly string[], prefix: string): string[] =>
|
|
1676
|
+
lines.map((line) => (line === "" ? prefix.trimEnd() : `${prefix}${line}`));
|
|
1677
|
+
|
|
1678
|
+
const mediaLines = (media: readonly CapturedMedia[]): string[] => {
|
|
1679
|
+
const lines: string[] = [];
|
|
1680
|
+
for (const item of media) {
|
|
1681
|
+
const fallback = item.kind === "gif" ? "GIF" : `${item.kind[0]?.toUpperCase() ?? ""}${item.kind.slice(1)}`;
|
|
1682
|
+
const label = escapeInline(item.title ?? item.alt ?? fallback);
|
|
1683
|
+
if (item.kind === "image" || item.kind === "gif") {
|
|
1684
|
+
lines.push(``);
|
|
1685
|
+
} else {
|
|
1686
|
+
if (item.previewUrl !== null) lines.push(``);
|
|
1687
|
+
lines.push(`- [${label}](<${item.url}>)`);
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
return lines;
|
|
1691
|
+
};
|
|
1692
|
+
|
|
1693
|
+
type RenderState = { count: number; readonly active: WeakSet<object> };
|
|
1694
|
+
|
|
1695
|
+
const unavailableLabel = (entry: CapturedUnavailableEntry): string => {
|
|
1696
|
+
const noun = entry.role === "comment" ? "comment" : entry.role === "quote" ? "quoted post" : "post";
|
|
1697
|
+
return `${entry.reason} ${noun} ${escapeInline(entry.id)}`;
|
|
1698
|
+
};
|
|
1699
|
+
|
|
1700
|
+
const renderQuote = (entry: CapturedEntry, depth: number, state: RenderState): string[] => {
|
|
1701
|
+
const lines = renderRootEntry(entry, depth + 1, state);
|
|
1702
|
+
return indentLines(lines, "> ");
|
|
1703
|
+
};
|
|
1704
|
+
|
|
1705
|
+
const renderReplies = (entries: readonly CapturedEntry[], depth: number, state: RenderState): string[] => {
|
|
1706
|
+
const lines: string[] = [];
|
|
1707
|
+
for (const entry of entries) lines.push(...renderNestedEntry(entry, depth, state));
|
|
1708
|
+
return lines;
|
|
1709
|
+
};
|
|
1710
|
+
|
|
1711
|
+
function renderNestedEntry(entry: CapturedEntry, depth: number, state: RenderState): string[] {
|
|
1712
|
+
const prefix = " ".repeat(depth);
|
|
1713
|
+
if (depth >= 64 || state.count >= 20_000) return [`${prefix}- *[render limit reached]*`];
|
|
1714
|
+
if (state.active.has(entry)) return [`${prefix}- *[cycle omitted]*`];
|
|
1715
|
+
state.count += 1;
|
|
1716
|
+
state.active.add(entry);
|
|
1717
|
+
let lines: string[];
|
|
1718
|
+
switch (entry.kind) {
|
|
1719
|
+
case "boundary":
|
|
1720
|
+
lines = [`${prefix}- *[${escapeInline(entry.detail)}]*`];
|
|
1721
|
+
break;
|
|
1722
|
+
case "more": {
|
|
1723
|
+
const count = entry.count === null ? "More comments" : `${entry.count} more comments`;
|
|
1724
|
+
const ids = entry.childIds.length === 0 ? "" : ` (${entry.childIds.map(escapeInline).join(", ")})`;
|
|
1725
|
+
lines = [`${prefix}- *${count}${ids}*`];
|
|
1726
|
+
break;
|
|
1727
|
+
}
|
|
1728
|
+
case "unavailable": {
|
|
1729
|
+
const source = entry.sourceUrl === null ? "" : ` · [source](<${entry.sourceUrl}>)`;
|
|
1730
|
+
lines = [`${prefix}- *[${unavailableLabel(entry)}]*${source}`];
|
|
1731
|
+
lines.push(...renderReplies(entry.replies, depth + 1, state));
|
|
1732
|
+
break;
|
|
1733
|
+
}
|
|
1734
|
+
case "content": {
|
|
1735
|
+
lines = [`${prefix}- ${metadata(entry)}`];
|
|
1736
|
+
if (entry.text.trim() !== "") {
|
|
1737
|
+
lines.push(`${prefix} `, ...indentLines(entry.text.trim().split("\n"), `${prefix} `));
|
|
1738
|
+
}
|
|
1739
|
+
const metricText = metricsLine(entry.metrics);
|
|
1740
|
+
if (metricText !== null) lines.push(`${prefix} `, `${prefix} _${metricText}_`);
|
|
1741
|
+
if (entry.media.length > 0) {
|
|
1742
|
+
lines.push(`${prefix} `, ...indentLines(mediaLines(entry.media), `${prefix} `));
|
|
1743
|
+
}
|
|
1744
|
+
for (const quote of entry.quotes) {
|
|
1745
|
+
lines.push(`${prefix} `, ...indentLines(renderQuote(quote, depth, state), `${prefix} `));
|
|
1746
|
+
}
|
|
1747
|
+
lines.push(...renderReplies(entry.replies, depth + 1, state));
|
|
1748
|
+
break;
|
|
1749
|
+
}
|
|
1750
|
+
}
|
|
1751
|
+
state.active.delete(entry);
|
|
1752
|
+
return lines;
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
function renderRootEntry(entry: CapturedEntry, depth: number, state: RenderState): string[] {
|
|
1756
|
+
if (depth >= 64 || state.count >= 20_000) return ["*[render limit reached]*"];
|
|
1757
|
+
if (state.active.has(entry)) return ["*[cycle omitted]*"];
|
|
1758
|
+
state.count += 1;
|
|
1759
|
+
state.active.add(entry);
|
|
1760
|
+
let lines: string[];
|
|
1761
|
+
switch (entry.kind) {
|
|
1762
|
+
case "boundary":
|
|
1763
|
+
lines = [`*[${escapeInline(entry.detail)}]*`];
|
|
1764
|
+
break;
|
|
1765
|
+
case "more": {
|
|
1766
|
+
const count = entry.count === null ? "More comments" : `${entry.count} more comments`;
|
|
1767
|
+
lines = [`*${count}*`];
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
case "unavailable": {
|
|
1771
|
+
const source = entry.sourceUrl === null ? "" : ` · [source](<${entry.sourceUrl}>)`;
|
|
1772
|
+
lines = [`*[${unavailableLabel(entry)}]*${source}`];
|
|
1773
|
+
if (entry.replies.length > 0) lines.push("", "#### Replies", "", ...renderReplies(entry.replies, 0, state));
|
|
1774
|
+
break;
|
|
1775
|
+
}
|
|
1776
|
+
case "content": {
|
|
1777
|
+
lines = [metadata(entry)];
|
|
1778
|
+
if (entry.text.trim() !== "") lines.push("", entry.text.trim());
|
|
1779
|
+
const metricText = metricsLine(entry.metrics);
|
|
1780
|
+
if (metricText !== null) lines.push("", `_${metricText}_`);
|
|
1781
|
+
if (entry.media.length > 0) lines.push("", ...mediaLines(entry.media));
|
|
1782
|
+
if (entry.quotes.length > 0) {
|
|
1783
|
+
lines.push("", "#### Quoted posts", "");
|
|
1784
|
+
for (const quote of entry.quotes) lines.push(...renderQuote(quote, depth, state), "");
|
|
1785
|
+
if (lines.at(-1) === "") lines.pop();
|
|
1786
|
+
}
|
|
1787
|
+
if (entry.replies.length > 0) lines.push("", "#### Replies", "", ...renderReplies(entry.replies, 0, state));
|
|
1788
|
+
break;
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
state.active.delete(entry);
|
|
1792
|
+
return lines;
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
/** Render a normalized document as stable, readable Markdown. */
|
|
1796
|
+
export function renderCapturedDocument(document: CapturedDocument): string {
|
|
1797
|
+
const lines = [
|
|
1798
|
+
`# ${cleanHeading(document.title) || "Captured post"}`,
|
|
1799
|
+
"",
|
|
1800
|
+
`Source: [${document.sourceUrl}](<${document.sourceUrl}>)`,
|
|
1801
|
+
`Platform: ${platformLabel(document.platform)}`,
|
|
1802
|
+
];
|
|
1803
|
+
const state: RenderState = { count: 0, active: new WeakSet<object>() };
|
|
1804
|
+
if (document.ancestors.length > 0) {
|
|
1805
|
+
lines.push("", "## Parent context", "");
|
|
1806
|
+
for (let index = 0; index < document.ancestors.length; index += 1) {
|
|
1807
|
+
lines.push(`### Parent ${index + 1}`, "", ...renderRootEntry(document.ancestors[index] ?? boundary("cycle", "Missing parent."), 0, state), "");
|
|
1808
|
+
}
|
|
1809
|
+
if (lines.at(-1) === "") lines.pop();
|
|
1810
|
+
}
|
|
1811
|
+
lines.push("", document.roots.length === 1 ? "## Post" : "## Posts", "");
|
|
1812
|
+
for (let index = 0; index < document.roots.length; index += 1) {
|
|
1813
|
+
if (document.roots.length > 1) lines.push(`### Post ${index + 1}`, "");
|
|
1814
|
+
const root = document.roots[index];
|
|
1815
|
+
if (root !== undefined) lines.push(...renderRootEntry(root, 0, state));
|
|
1816
|
+
if (index < document.roots.length - 1) lines.push("");
|
|
1817
|
+
}
|
|
1818
|
+
if (document.warnings.length > 0) {
|
|
1819
|
+
lines.push("", "## Capture notes", "");
|
|
1820
|
+
for (const warning of document.warnings) lines.push(`- ${warning}`);
|
|
1821
|
+
}
|
|
1822
|
+
const markdown = `${lines.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd()}\n`;
|
|
1823
|
+
return rewriteContent(markdown, new URL(document.sourceUrl), new Map(), { remoteImages: "embed" });
|
|
1824
|
+
}
|