feedkeeper 0.0.0-stage → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +22 -2
- package/package.json +57 -4
- package/server/dist/api/archive.js +19 -0
- package/server/dist/api/auth.js +167 -0
- package/server/dist/api/feeds.js +207 -0
- package/server/dist/api/filters.js +32 -0
- package/server/dist/api/folders.js +53 -0
- package/server/dist/api/index.js +37 -0
- package/server/dist/api/items.js +129 -0
- package/server/dist/api/oauth.js +14 -0
- package/server/dist/api/onboarding.js +30 -0
- package/server/dist/api/push.js +51 -0
- package/server/dist/api/system.js +42 -0
- package/server/dist/api/tokens.js +28 -0
- package/server/dist/api/v1/index.js +178 -0
- package/server/dist/api/v1/resources.js +653 -0
- package/server/dist/api/v1/serialize.js +48 -0
- package/server/dist/auth/avatar.js +13 -0
- package/server/dist/auth/capabilities.js +53 -0
- package/server/dist/auth/middleware.js +77 -0
- package/server/dist/auth/pairing.js +36 -0
- package/server/dist/auth/password.js +21 -0
- package/server/dist/auth/security.js +126 -0
- package/server/dist/auth/securityRoutes.js +249 -0
- package/server/dist/auth/tokens.js +64 -0
- package/server/dist/auth/users.js +54 -0
- package/server/dist/backup-cli.js +101 -0
- package/server/dist/cli.js +135 -0
- package/server/dist/clientAddress.js +26 -0
- package/server/dist/config.js +51 -0
- package/server/dist/db/index.js +40 -0
- package/server/dist/db/migrations/0001_init.sql +70 -0
- package/server/dist/db/migrations/0002_feed_health.sql +5 -0
- package/server/dist/db/migrations/0003_retention.sql +13 -0
- package/server/dist/db/migrations/0004_bookmarks_and_filters.sql +20 -0
- package/server/dist/db/migrations/0005_folders.sql +15 -0
- package/server/dist/db/migrations/0006_feed_position.sql +5 -0
- package/server/dist/db/migrations/0007_item_image_url.sql +2 -0
- package/server/dist/db/migrations/0008_item_content_html.sql +2 -0
- package/server/dist/db/migrations/0009_item_full_content_html.sql +1 -0
- package/server/dist/db/migrations/0010_token_scope.sql +3 -0
- package/server/dist/db/migrations/0011_user_avatars.sql +8 -0
- package/server/dist/db/migrations/0012_full_text_settings.sql +5 -0
- package/server/dist/db/migrations/0013_feed_icons.sql +4 -0
- package/server/dist/db/migrations/0014_bookmark_archive.sql +16 -0
- package/server/dist/db/migrations/0015_push.sql +13 -0
- package/server/dist/db/migrations/0016_feed_badge.sql +2 -0
- package/server/dist/db/migrations/0017_devices.sql +13 -0
- package/server/dist/db/migrations/0018_change_log.sql +166 -0
- package/server/dist/db/migrations/0019_native_extensions.sql +111 -0
- package/server/dist/db/migrations/0020_fts.sql +32 -0
- package/server/dist/db/migrations/0021_note_revisions.sql +33 -0
- package/server/dist/db/migrations/0022_high_resolution_icons.sql +2 -0
- package/server/dist/db/migrations/0023_server_editions.sql +81 -0
- package/server/dist/db/migrations/0024_inbox.sql +5 -0
- package/server/dist/db/migrations/0025_edition_overview.sql +2 -0
- package/server/dist/db/migrations/0026_article_content.sql +43 -0
- package/server/dist/db/migrations/0027_oauth.sql +38 -0
- package/server/dist/db/migrations/0028_user_intro.sql +2 -0
- package/server/dist/db/migrations/0029_account_security.sql +35 -0
- package/server/dist/db/migrations/0030_feed_poll_jobs.sql +15 -0
- package/server/dist/db/migrations/0031_push_outbox.sql +19 -0
- package/server/dist/db/migrations/0032_normalize_published_at.sql +12 -0
- package/server/dist/feedkeeper.js +38 -0
- package/server/dist/feeds/archive.js +334 -0
- package/server/dist/feeds/articleContent.js +26 -0
- package/server/dist/feeds/cleanup.js +179 -0
- package/server/dist/feeds/dates.js +21 -0
- package/server/dist/feeds/discovery.js +153 -0
- package/server/dist/feeds/extractor.js +208 -0
- package/server/dist/feeds/feedIcon.js +171 -0
- package/server/dist/feeds/fetcher.js +217 -0
- package/server/dist/feeds/fullText.js +91 -0
- package/server/dist/feeds/fullTextQueue.js +159 -0
- package/server/dist/feeds/icon.js +97 -0
- package/server/dist/feeds/imageCache.js +266 -0
- package/server/dist/feeds/imageCacheSettings.js +19 -0
- package/server/dist/feeds/imageTypes.js +16 -0
- package/server/dist/feeds/inbox.js +128 -0
- package/server/dist/feeds/opml.js +190 -0
- package/server/dist/feeds/pollQueue.js +301 -0
- package/server/dist/feeds/pollSettings.js +23 -0
- package/server/dist/feeds/poller.js +223 -0
- package/server/dist/feeds/repository.js +687 -0
- package/server/dist/feeds/requestGate.js +99 -0
- package/server/dist/feeds/service.js +162 -0
- package/server/dist/feeds/ssrfGuard.js +98 -0
- package/server/dist/feeds/text.js +35 -0
- package/server/dist/mcp/digest.js +73 -0
- package/server/dist/mcp/http.js +46 -0
- package/server/dist/mcp/items.js +73 -0
- package/server/dist/mcp/promptsAndResources.js +43 -0
- package/server/dist/mcp/server.js +547 -0
- package/server/dist/native/editionSelection.js +127 -0
- package/server/dist/native/editionText.js +15 -0
- package/server/dist/native/editions.js +218 -0
- package/server/dist/native/preferences.js +66 -0
- package/server/dist/native/requests.js +35 -0
- package/server/dist/oauth/constants.js +6 -0
- package/server/dist/oauth/host.js +8 -0
- package/server/dist/oauth/pages.js +148 -0
- package/server/dist/oauth/routes.js +370 -0
- package/server/dist/oauth/store.js +104 -0
- package/server/dist/push.js +147 -0
- package/server/dist/pushOutbox.js +164 -0
- package/server/dist/security-recover-cli.js +29 -0
- package/server/dist/server.js +108 -0
- package/server/dist/setup-cli.js +97 -0
- package/server/dist/sync/changeLog.js +170 -0
- package/server/dist/sync/mutations.js +144 -0
- package/server/dist/trustProxy.js +14 -0
- package/server/dist/version.js +6 -0
- package/server/package.json +6 -0
- package/web/dist/app-icon-180.png +0 -0
- package/web/dist/app-icon-192.png +0 -0
- package/web/dist/app-icon-512.png +0 -0
- package/web/dist/assets/index-B4e2oqbo.css +2 -0
- package/web/dist/assets/index-xx6TyOwB.js +32 -0
- package/web/dist/assets/inter-latin-400-normal-C38fXH4l.woff2 +0 -0
- package/web/dist/assets/inter-latin-400-normal-CyCys3Eg.woff +0 -0
- package/web/dist/assets/inter-latin-500-normal-BL9OpVg8.woff +0 -0
- package/web/dist/assets/inter-latin-500-normal-Cerq10X2.woff2 +0 -0
- package/web/dist/assets/inter-latin-600-normal-CiBQ2DWP.woff +0 -0
- package/web/dist/assets/inter-latin-600-normal-LgqL8muc.woff2 +0 -0
- package/web/dist/assets/manrope-latin-600-normal-4f0koTD-.woff2 +0 -0
- package/web/dist/assets/manrope-latin-600-normal-BqgrALkZ.woff +0 -0
- package/web/dist/assets/manrope-latin-700-normal-BZp_XxE4.woff2 +0 -0
- package/web/dist/assets/manrope-latin-700-normal-DGRFkw-m.woff +0 -0
- package/web/dist/favicon-32.png +0 -0
- package/web/dist/index.html +40 -0
- package/web/dist/logo.svg +50 -0
- package/web/dist/manifest.webmanifest +64 -0
- package/web/dist/robots.txt +3 -0
- package/web/dist/sw.js +198 -0
- package/web/dist/theme-init.js +10 -0
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { link, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
|
|
3
|
+
import { join, resolve } from "node:path";
|
|
4
|
+
import { FeedHttpError } from "./fetcher.js";
|
|
5
|
+
import { detectImageType } from "./imageTypes.js";
|
|
6
|
+
import { readPollSettings } from "./pollSettings.js";
|
|
7
|
+
const MAX_IMAGE_BYTES = 5 * 1024 * 1024;
|
|
8
|
+
const KEY = /^[0-9a-f]{64}$/;
|
|
9
|
+
export class ImageCacheBusyError extends Error {
|
|
10
|
+
constructor() { super("Image cache is busy"); }
|
|
11
|
+
}
|
|
12
|
+
/** A disposable disk cache, shared only within a feed. One process owns its directory. */
|
|
13
|
+
export class ImageCache {
|
|
14
|
+
settings;
|
|
15
|
+
now;
|
|
16
|
+
directory;
|
|
17
|
+
entries = new Map();
|
|
18
|
+
negative = new Map();
|
|
19
|
+
running = new Map();
|
|
20
|
+
ready;
|
|
21
|
+
waiting = [];
|
|
22
|
+
active = 0;
|
|
23
|
+
writes = Promise.resolve();
|
|
24
|
+
stopping = false;
|
|
25
|
+
diskAvailable = true;
|
|
26
|
+
bytes = 0;
|
|
27
|
+
counters = { hits: 0, misses: 0, shared: 0, negativeHits: 0, busy: 0, errors: 0, storageErrors: 0, evictions: 0 };
|
|
28
|
+
constructor(settings, now = Date.now) {
|
|
29
|
+
this.settings = settings;
|
|
30
|
+
this.now = now;
|
|
31
|
+
this.directory = resolve(settings.path);
|
|
32
|
+
this.ready = this.restore().catch(() => { this.diskAvailable = false; this.counters.storageErrors++; });
|
|
33
|
+
}
|
|
34
|
+
metrics() { return { ...this.counters, entries: this.entries.size, bytes: this.bytes, running: this.active, pending: this.waiting.length }; }
|
|
35
|
+
async get(scope, url, load) {
|
|
36
|
+
if (this.stopping)
|
|
37
|
+
throw new ImageCacheBusyError();
|
|
38
|
+
const key = createHash("sha256").update(`${scope}\0${url}`).digest("hex");
|
|
39
|
+
const shared = this.running.get(key);
|
|
40
|
+
if (shared) {
|
|
41
|
+
this.counters.shared++;
|
|
42
|
+
return shared.promise;
|
|
43
|
+
}
|
|
44
|
+
if (this.active >= this.settings.concurrency && this.waiting.length >= this.settings.maxWaiters) {
|
|
45
|
+
this.counters.busy++;
|
|
46
|
+
throw new ImageCacheBusyError();
|
|
47
|
+
}
|
|
48
|
+
const controller = new AbortController();
|
|
49
|
+
let complete;
|
|
50
|
+
let fail;
|
|
51
|
+
let finishWork;
|
|
52
|
+
const promise = new Promise((resolve, reject) => { complete = resolve; fail = reject; });
|
|
53
|
+
const finished = new Promise((resolve) => { finishWork = resolve; });
|
|
54
|
+
let started = false;
|
|
55
|
+
const deadline = setTimeout(() => controller.abort(new DOMException("Image deadline exceeded", "TimeoutError")), readPollSettings().requestTimeoutMs);
|
|
56
|
+
deadline.unref();
|
|
57
|
+
this.running.set(key, { promise, finished, controller });
|
|
58
|
+
const cancelWaiting = () => {
|
|
59
|
+
fail(controller.signal.reason);
|
|
60
|
+
if (started)
|
|
61
|
+
return; // Keep the slot until any outstanding I/O has actually stopped.
|
|
62
|
+
const index = this.waiting.indexOf(job);
|
|
63
|
+
if (index >= 0)
|
|
64
|
+
this.waiting.splice(index, 1);
|
|
65
|
+
this.running.delete(key);
|
|
66
|
+
clearTimeout(deadline);
|
|
67
|
+
finishWork();
|
|
68
|
+
};
|
|
69
|
+
const job = { run: () => {
|
|
70
|
+
started = true;
|
|
71
|
+
this.active++;
|
|
72
|
+
const finish = () => {
|
|
73
|
+
clearTimeout(deadline);
|
|
74
|
+
controller.signal.removeEventListener("abort", cancelWaiting);
|
|
75
|
+
this.active--;
|
|
76
|
+
this.running.delete(key);
|
|
77
|
+
finishWork();
|
|
78
|
+
this.pump();
|
|
79
|
+
};
|
|
80
|
+
void this.resolve(key, load, controller.signal).then((image) => { finish(); complete(image); }, (error) => { finish(); fail(error); });
|
|
81
|
+
} };
|
|
82
|
+
controller.signal.addEventListener("abort", cancelWaiting, { once: true });
|
|
83
|
+
this.waiting.push(job);
|
|
84
|
+
this.pump();
|
|
85
|
+
return promise;
|
|
86
|
+
}
|
|
87
|
+
pump() {
|
|
88
|
+
while (this.active < this.settings.concurrency && this.waiting.length)
|
|
89
|
+
this.waiting.shift().run();
|
|
90
|
+
}
|
|
91
|
+
async close() {
|
|
92
|
+
this.stopping = true;
|
|
93
|
+
const jobs = [...this.running.values()];
|
|
94
|
+
for (const { controller } of jobs)
|
|
95
|
+
controller.abort();
|
|
96
|
+
await this.ready;
|
|
97
|
+
await Promise.allSettled(jobs.map(({ finished }) => finished));
|
|
98
|
+
await this.writes;
|
|
99
|
+
await this.serialize(() => this.saveIndex()).catch(() => undefined);
|
|
100
|
+
}
|
|
101
|
+
async prune() {
|
|
102
|
+
await this.ready;
|
|
103
|
+
await this.serialize(async () => { await this.trim(); await this.saveIndex(); });
|
|
104
|
+
}
|
|
105
|
+
serialize(work) {
|
|
106
|
+
const result = this.writes.then(work);
|
|
107
|
+
this.writes = result.catch(() => { this.diskAvailable = false; this.counters.storageErrors++; });
|
|
108
|
+
return result;
|
|
109
|
+
}
|
|
110
|
+
file(key) { return join(this.directory, `${key}.img`); }
|
|
111
|
+
async restore() {
|
|
112
|
+
if (!this.settings.maxBytes) {
|
|
113
|
+
this.diskAvailable = false;
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
await mkdir(this.directory, { recursive: true, mode: 0o700 });
|
|
117
|
+
const index = join(this.directory, "index.json");
|
|
118
|
+
let saved = [];
|
|
119
|
+
try {
|
|
120
|
+
if ((await stat(index)).size > 8 * 1024 * 1024)
|
|
121
|
+
throw new Error("Oversized image cache index");
|
|
122
|
+
saved = JSON.parse(await readFile(index, "utf8"));
|
|
123
|
+
}
|
|
124
|
+
catch { /* Missing or interrupted cache metadata is safe to discard. */ }
|
|
125
|
+
if (Array.isArray(saved)) {
|
|
126
|
+
for (const row of saved.slice(0, 20_000)) {
|
|
127
|
+
if (!row || typeof row !== "object" || !KEY.test(row.key) || !Number.isSafeInteger(row.size) || row.size <= 0 || row.size > MAX_IMAGE_BYTES ||
|
|
128
|
+
!Number.isFinite(row.expires) || !Number.isFinite(row.used) || row.expires <= this.now() || this.entries.has(row.key))
|
|
129
|
+
continue;
|
|
130
|
+
try {
|
|
131
|
+
if ((await stat(this.file(row.key))).size !== row.size)
|
|
132
|
+
continue;
|
|
133
|
+
this.entries.set(row.key, row);
|
|
134
|
+
this.bytes += row.size;
|
|
135
|
+
}
|
|
136
|
+
catch { /* Missing files are cache misses. */ }
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
await this.trim();
|
|
140
|
+
for (const file of await readdir(this.directory)) {
|
|
141
|
+
if ((/^[0-9a-f]{64}\.img$/.test(file) && !this.entries.has(file.slice(0, 64))) || /^[0-9a-f]{64}\.tmp$/.test(file) || file === "index.tmp") {
|
|
142
|
+
await unlink(join(this.directory, file));
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
await this.saveIndex();
|
|
146
|
+
}
|
|
147
|
+
async resolve(key, load, signal) {
|
|
148
|
+
await this.ready;
|
|
149
|
+
signal.throwIfAborted();
|
|
150
|
+
const missing = this.negative.get(key);
|
|
151
|
+
if (missing && missing.expires > this.now()) {
|
|
152
|
+
this.counters.negativeHits++;
|
|
153
|
+
if (missing.result === "unsupported")
|
|
154
|
+
return "unsupported";
|
|
155
|
+
throw new FeedHttpError(missing.result, undefined, "");
|
|
156
|
+
}
|
|
157
|
+
this.negative.delete(key);
|
|
158
|
+
const entry = this.entries.get(key);
|
|
159
|
+
if (entry && entry.expires > this.now()) {
|
|
160
|
+
try {
|
|
161
|
+
const buffer = await readFile(this.file(key), { signal });
|
|
162
|
+
const type = detectImageType(buffer);
|
|
163
|
+
if (buffer.length === entry.size && type) {
|
|
164
|
+
entry.used = this.now();
|
|
165
|
+
this.counters.hits++;
|
|
166
|
+
return { buffer, mime: type.mime, path: this.file(key) };
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
catch { /* Re-fetch evicted or damaged files through the guarded downloader. */ }
|
|
170
|
+
}
|
|
171
|
+
this.counters.misses++;
|
|
172
|
+
try {
|
|
173
|
+
signal.throwIfAborted();
|
|
174
|
+
const image = await load(signal);
|
|
175
|
+
signal.throwIfAborted();
|
|
176
|
+
if (image.buffer.length > MAX_IMAGE_BYTES)
|
|
177
|
+
throw new Error("Image exceeds the allowed size");
|
|
178
|
+
const type = detectImageType(image.buffer);
|
|
179
|
+
if (!type) {
|
|
180
|
+
this.rememberNegative(key, "unsupported");
|
|
181
|
+
return "unsupported";
|
|
182
|
+
}
|
|
183
|
+
const result = { buffer: image.buffer, mime: type.mime, path: image.path };
|
|
184
|
+
const ttl = Math.min(this.settings.ttlMs, image.cacheTtlMs ?? this.settings.ttlMs);
|
|
185
|
+
if (this.diskAvailable && ttl > 0 && image.buffer.length <= this.settings.maxBytes) {
|
|
186
|
+
try {
|
|
187
|
+
await this.serialize(async () => {
|
|
188
|
+
if (signal.aborted || !this.diskAvailable)
|
|
189
|
+
return;
|
|
190
|
+
await this.remove(key);
|
|
191
|
+
await this.trim(image.buffer.length, 1);
|
|
192
|
+
const temporary = join(this.directory, `${key}.tmp`);
|
|
193
|
+
await unlink(temporary).catch((error) => { if (error.code !== "ENOENT")
|
|
194
|
+
throw error; });
|
|
195
|
+
// Hard links let the cache and permanent archive share the same bytes.
|
|
196
|
+
if (image.path) {
|
|
197
|
+
await link(image.path, temporary).catch(() => writeFile(temporary, image.buffer, { mode: 0o600 }));
|
|
198
|
+
}
|
|
199
|
+
else
|
|
200
|
+
await writeFile(temporary, image.buffer, { mode: 0o600 });
|
|
201
|
+
await rename(temporary, this.file(key));
|
|
202
|
+
this.entries.set(key, { key, size: image.buffer.length, expires: this.now() + ttl, used: this.now() });
|
|
203
|
+
this.bytes += image.buffer.length;
|
|
204
|
+
await this.saveIndex();
|
|
205
|
+
result.path = this.file(key);
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
catch {
|
|
209
|
+
await unlink(join(this.directory, `${key}.tmp`)).catch(() => undefined);
|
|
210
|
+
// A full or read-only cache must not break image delivery.
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
return result;
|
|
214
|
+
}
|
|
215
|
+
catch (error) {
|
|
216
|
+
this.counters.errors++;
|
|
217
|
+
if (error instanceof FeedHttpError && (error.status === 404 || error.status === 410))
|
|
218
|
+
this.rememberNegative(key, error.status);
|
|
219
|
+
throw error;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
rememberNegative(key, result) {
|
|
223
|
+
if (!this.settings.negativeTtlMs)
|
|
224
|
+
return;
|
|
225
|
+
for (const [key, row] of this.negative)
|
|
226
|
+
if (row.expires <= this.now())
|
|
227
|
+
this.negative.delete(key);
|
|
228
|
+
while (this.negative.size >= this.settings.maxEntries)
|
|
229
|
+
this.negative.delete(this.negative.keys().next().value);
|
|
230
|
+
this.negative.set(key, { result, expires: this.now() + this.settings.negativeTtlMs });
|
|
231
|
+
}
|
|
232
|
+
async remove(key) {
|
|
233
|
+
const entry = this.entries.get(key);
|
|
234
|
+
if (!entry)
|
|
235
|
+
return;
|
|
236
|
+
// Do not forget a file we could not remove: its bytes still count towards the limit.
|
|
237
|
+
try {
|
|
238
|
+
await unlink(this.file(key));
|
|
239
|
+
}
|
|
240
|
+
catch (error) {
|
|
241
|
+
if (error.code !== "ENOENT")
|
|
242
|
+
throw error;
|
|
243
|
+
}
|
|
244
|
+
this.entries.delete(key);
|
|
245
|
+
this.bytes -= entry.size;
|
|
246
|
+
this.counters.evictions++;
|
|
247
|
+
}
|
|
248
|
+
async trim(extraBytes = 0, extraEntries = 0) {
|
|
249
|
+
const oldest = [...this.entries.values()].sort((a, b) => a.used - b.used);
|
|
250
|
+
for (const entry of oldest)
|
|
251
|
+
if (entry.expires <= this.now())
|
|
252
|
+
await this.remove(entry.key);
|
|
253
|
+
for (const entry of oldest) {
|
|
254
|
+
if (this.bytes + extraBytes <= this.settings.maxBytes && this.entries.size + extraEntries <= this.settings.maxEntries)
|
|
255
|
+
break;
|
|
256
|
+
await this.remove(entry.key);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
async saveIndex() {
|
|
260
|
+
if (!this.diskAvailable)
|
|
261
|
+
return;
|
|
262
|
+
await writeFile(join(this.directory, "index.tmp"), JSON.stringify([...this.entries.values()]), { mode: 0o600 });
|
|
263
|
+
await rename(join(this.directory, "index.tmp"), join(this.directory, "index.json"));
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
//# sourceMappingURL=imageCache.js.map
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { dirname, join } from "node:path";
|
|
2
|
+
function integer(env, name, fallback, min, max) {
|
|
3
|
+
const value = env[name] === undefined ? fallback : Number(env[name]);
|
|
4
|
+
if (!Number.isSafeInteger(value) || value < min || value > max)
|
|
5
|
+
throw new Error(`${name} must be an integer between ${min} and ${max}`);
|
|
6
|
+
return value;
|
|
7
|
+
}
|
|
8
|
+
export function readImageCacheSettings(env = process.env) {
|
|
9
|
+
return {
|
|
10
|
+
path: env.IMAGE_CACHE_PATH ?? join(dirname(env.DATABASE_PATH ?? "./data/feedkeeper.sqlite"), "image-cache"),
|
|
11
|
+
maxBytes: integer(env, "IMAGE_CACHE_MAX_BYTES", 256 * 1024 * 1024, 0, 1024 ** 4),
|
|
12
|
+
maxEntries: integer(env, "IMAGE_CACHE_MAX_ENTRIES", 2_000, 1, 20_000),
|
|
13
|
+
ttlMs: integer(env, "IMAGE_CACHE_TTL_SECONDS", 21_600, 1, 604_800) * 1_000,
|
|
14
|
+
negativeTtlMs: integer(env, "IMAGE_CACHE_NEGATIVE_TTL_SECONDS", 60, 0, 300) * 1_000,
|
|
15
|
+
concurrency: integer(env, "IMAGE_CACHE_CONCURRENCY", 4, 1, 32),
|
|
16
|
+
maxWaiters: integer(env, "IMAGE_CACHE_MAX_WAITERS", 32, 0, 256),
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=imageCacheSettings.js.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** SVG is excluded because it can carry scripts. */
|
|
2
|
+
export function detectImageType(data) {
|
|
3
|
+
const ascii = (start, end) => String.fromCharCode(...data.subarray(start, end));
|
|
4
|
+
if (data.length >= 3 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff)
|
|
5
|
+
return { mime: "image/jpeg", ext: "jpg" };
|
|
6
|
+
if (data.length >= 8 && ascii(1, 4) === "PNG" && data[0] === 0x89)
|
|
7
|
+
return { mime: "image/png", ext: "png" };
|
|
8
|
+
if (data.length >= 12 && ascii(0, 4) === "RIFF" && ascii(8, 12) === "WEBP")
|
|
9
|
+
return { mime: "image/webp", ext: "webp" };
|
|
10
|
+
if (data.length >= 6 && (ascii(0, 6) === "GIF87a" || ascii(0, 6) === "GIF89a"))
|
|
11
|
+
return { mime: "image/gif", ext: "gif" };
|
|
12
|
+
if (data.length >= 12 && ascii(4, 8) === "ftyp" && ["avif", "avis"].includes(ascii(8, 12)))
|
|
13
|
+
return { mime: "image/avif", ext: "avif" };
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=imageTypes.js.map
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { db } from "../db/index.js";
|
|
3
|
+
import { extractArticle } from "./extractor.js";
|
|
4
|
+
import { assertPublicHttpUrl, normalizeUrlCandidate } from "./ssrfGuard.js";
|
|
5
|
+
import { articleImageUrls, scheduleArchive } from "./archive.js";
|
|
6
|
+
import { findFeedById, findFeedByUrl, isUserSubscribed, listSubscriptionsForUser, subscribe, bookmarkItem, findItemById, findItemForUser, setItemNote, } from "./repository.js";
|
|
7
|
+
function escapeHtml(str) {
|
|
8
|
+
return str
|
|
9
|
+
.replace(/&/g, "&")
|
|
10
|
+
.replace(/</g, "<")
|
|
11
|
+
.replace(/>/g, ">")
|
|
12
|
+
.replace(/"/g, """)
|
|
13
|
+
.replace(/'/g, "'");
|
|
14
|
+
}
|
|
15
|
+
export function ensureUserInbox(userId) {
|
|
16
|
+
const inboxUrl = `feedkeeper://inbox/${userId}`;
|
|
17
|
+
let feed = findFeedByUrl(inboxUrl);
|
|
18
|
+
if (!feed) {
|
|
19
|
+
db.prepare(`INSERT INTO feeds (url, title, is_system_inbox, poll_interval_minutes)
|
|
20
|
+
VALUES (?, 'Inbox', 1, 0)
|
|
21
|
+
ON CONFLICT (url) DO UPDATE SET is_system_inbox = 1`).run(inboxUrl);
|
|
22
|
+
feed = findFeedByUrl(inboxUrl);
|
|
23
|
+
}
|
|
24
|
+
else if (!feed.is_system_inbox) {
|
|
25
|
+
db.prepare("UPDATE feeds SET is_system_inbox = 1 WHERE id = ?").run(feed.id);
|
|
26
|
+
feed = findFeedById(feed.id);
|
|
27
|
+
}
|
|
28
|
+
if (!isUserSubscribed(userId, feed.id)) {
|
|
29
|
+
subscribe(userId, feed.id, "Inbox");
|
|
30
|
+
}
|
|
31
|
+
const sub = listSubscriptionsForUser(userId).find((s) => s.id === feed.id);
|
|
32
|
+
if (!sub) {
|
|
33
|
+
throw new Error("Failed to ensure user inbox subscription");
|
|
34
|
+
}
|
|
35
|
+
return sub;
|
|
36
|
+
}
|
|
37
|
+
export async function saveToInbox(userId, input) {
|
|
38
|
+
let targetUrl = null;
|
|
39
|
+
let title = input.title?.trim() || null;
|
|
40
|
+
let contentHtml = input.contentHtml?.trim() || null;
|
|
41
|
+
let snippet = null;
|
|
42
|
+
let extractionStatus = contentHtml ? "ready" : "pending";
|
|
43
|
+
let extractedPartial = false;
|
|
44
|
+
let imageUrl = null;
|
|
45
|
+
if (input.url?.trim()) {
|
|
46
|
+
const normalized = normalizeUrlCandidate(input.url.trim());
|
|
47
|
+
await assertPublicHttpUrl(normalized);
|
|
48
|
+
targetUrl = normalized;
|
|
49
|
+
if (!contentHtml) {
|
|
50
|
+
try {
|
|
51
|
+
const extracted = await extractArticle(targetUrl);
|
|
52
|
+
if (extracted.status === "ok" || extracted.status === "partial") {
|
|
53
|
+
extractionStatus = extracted.status === "ok" ? "ready" : "partial";
|
|
54
|
+
extractedPartial = extracted.status === "partial";
|
|
55
|
+
if (!title && extracted.article.title) {
|
|
56
|
+
title = extracted.article.title;
|
|
57
|
+
}
|
|
58
|
+
contentHtml = extracted.article.contentHtml;
|
|
59
|
+
snippet = extracted.article.excerpt ?? (extracted.article.textContent ? extracted.article.textContent.slice(0, 300) : null);
|
|
60
|
+
const images = articleImageUrls(contentHtml, targetUrl);
|
|
61
|
+
imageUrl = images[0] ?? null;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
console.warn(`[inbox] Article extraction failed for ${targetUrl}:`, err);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (!contentHtml && input.textContent?.trim()) {
|
|
70
|
+
const cleanText = input.textContent.trim();
|
|
71
|
+
contentHtml = cleanText
|
|
72
|
+
.split(/\n\s*\n/)
|
|
73
|
+
.map((p) => `<p>${escapeHtml(p.trim())}</p>`)
|
|
74
|
+
.join("\n");
|
|
75
|
+
if (!snippet) {
|
|
76
|
+
snippet = cleanText.slice(0, 300);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
if (!title) {
|
|
80
|
+
title = targetUrl ?? (snippet ? snippet.slice(0, 80) : "Saved Clipping");
|
|
81
|
+
}
|
|
82
|
+
const inbox = ensureUserInbox(userId);
|
|
83
|
+
const now = new Date().toISOString();
|
|
84
|
+
const guid = targetUrl ? targetUrl : `urn:feedkeeper:inbox:${randomUUID()}`;
|
|
85
|
+
const row = db
|
|
86
|
+
.prepare(`INSERT INTO items (feed_id, guid, title, link, content_snippet, content_html, full_content_html, image_url, published_at, created_at, extraction_status, extraction_retry_at)
|
|
87
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
88
|
+
ON CONFLICT (feed_id, guid) DO UPDATE SET
|
|
89
|
+
title = COALESCE(excluded.title, items.title),
|
|
90
|
+
link = COALESCE(excluded.link, items.link),
|
|
91
|
+
content_snippet = COALESCE(excluded.content_snippet, items.content_snippet),
|
|
92
|
+
content_html = COALESCE(excluded.content_html, items.content_html),
|
|
93
|
+
full_content_html = COALESCE(excluded.full_content_html, items.full_content_html),
|
|
94
|
+
image_url = COALESCE(excluded.image_url, items.image_url),
|
|
95
|
+
extraction_status = CASE WHEN COALESCE(excluded.full_content_html, items.full_content_html) IS NOT NULL THEN 'ready' ELSE excluded.extraction_status END,
|
|
96
|
+
extraction_retry_at = excluded.extraction_retry_at
|
|
97
|
+
RETURNING id`)
|
|
98
|
+
.get(inbox.id, guid, title, targetUrl, snippet, contentHtml, extractedPartial ? null : contentHtml, imageUrl, now, now, contentHtml && !extractedPartial ? "ready" : extractionStatus, extractedPartial ? new Date(Date.now() + 86400_000).toISOString() : null);
|
|
99
|
+
const itemId = row.id;
|
|
100
|
+
bookmarkItem(userId, itemId);
|
|
101
|
+
let hasNote = false;
|
|
102
|
+
if (input.note?.trim()) {
|
|
103
|
+
setItemNote(userId, itemId, input.note.trim());
|
|
104
|
+
hasNote = true;
|
|
105
|
+
}
|
|
106
|
+
scheduleArchive(itemId);
|
|
107
|
+
const item = findItemForUser(userId, itemId);
|
|
108
|
+
if (!item) {
|
|
109
|
+
throw new Error("Item was saved but could not be retrieved");
|
|
110
|
+
}
|
|
111
|
+
return { itemId, item, hasNote };
|
|
112
|
+
}
|
|
113
|
+
export function deleteInboxItem(userId, itemId) {
|
|
114
|
+
const inbox = ensureUserInbox(userId);
|
|
115
|
+
const item = findItemById(itemId);
|
|
116
|
+
if (!item || item.feed_id !== inbox.id) {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
db.transaction(() => {
|
|
120
|
+
db.prepare("DELETE FROM item_bookmarks WHERE item_id = ?").run(itemId);
|
|
121
|
+
db.prepare("DELETE FROM item_reads WHERE item_id = ?").run(itemId);
|
|
122
|
+
db.prepare("DELETE FROM item_progress WHERE item_id = ?").run(itemId);
|
|
123
|
+
db.prepare("DELETE FROM item_notes WHERE item_id = ?").run(itemId);
|
|
124
|
+
db.prepare("DELETE FROM items WHERE id = ?").run(itemId);
|
|
125
|
+
})();
|
|
126
|
+
return true;
|
|
127
|
+
}
|
|
128
|
+
//# sourceMappingURL=inbox.js.map
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { XMLParser } from "fast-xml-parser";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { config } from "../config.js";
|
|
4
|
+
import { listSubscriptionsForUser, findFeedByUrl, createFeed, isUserSubscribed, subscribe, createFolder, } from "./repository.js";
|
|
5
|
+
import { assertPublicHttpUrl, SsrfBlockedError } from "./ssrfGuard.js";
|
|
6
|
+
import { requestFeedPoll } from "./poller.js";
|
|
7
|
+
import { decodeEntities } from "./text.js";
|
|
8
|
+
function escapeXml(unsafe) {
|
|
9
|
+
return unsafe.replace(/[<>&'"]/g, (char) => {
|
|
10
|
+
switch (char) {
|
|
11
|
+
case "<":
|
|
12
|
+
return "<";
|
|
13
|
+
case ">":
|
|
14
|
+
return ">";
|
|
15
|
+
case "&":
|
|
16
|
+
return "&";
|
|
17
|
+
case "'":
|
|
18
|
+
return "'";
|
|
19
|
+
case '"':
|
|
20
|
+
return """;
|
|
21
|
+
default:
|
|
22
|
+
return char;
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
function feedToOutlineXml(s, indent = " ") {
|
|
27
|
+
const text = escapeXml(s.label || s.title || s.url);
|
|
28
|
+
const xmlUrl = escapeXml(s.url);
|
|
29
|
+
const htmlUrl = s.site_url ? ` htmlUrl="${escapeXml(s.site_url)}"` : "";
|
|
30
|
+
return `${indent}<outline text="${text}" title="${text}" type="rss" xmlUrl="${xmlUrl}"${htmlUrl} />`;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Generates standard OPML 2.0 XML representation of all subscribed feeds for a user.
|
|
34
|
+
*/
|
|
35
|
+
export function generateOpml(userId) {
|
|
36
|
+
const subscriptions = listSubscriptionsForUser(userId).filter((s) => !s.is_system_inbox);
|
|
37
|
+
// Group feeds by folder
|
|
38
|
+
const unfiled = [];
|
|
39
|
+
const folders = new Map();
|
|
40
|
+
for (const sub of subscriptions) {
|
|
41
|
+
if (sub.folder_name) {
|
|
42
|
+
const list = folders.get(sub.folder_name) ?? [];
|
|
43
|
+
list.push(sub);
|
|
44
|
+
folders.set(sub.folder_name, list);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
unfiled.push(sub);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const sections = [];
|
|
51
|
+
// Add folder outlines
|
|
52
|
+
for (const [folderName, feeds] of folders.entries()) {
|
|
53
|
+
const escapedName = escapeXml(folderName);
|
|
54
|
+
const feedOutlines = feeds.map((f) => feedToOutlineXml(f, " ")).join("\n");
|
|
55
|
+
sections.push(` <outline text="${escapedName}" title="${escapedName}">\n${feedOutlines}\n </outline>`);
|
|
56
|
+
}
|
|
57
|
+
// Add unfiled feeds
|
|
58
|
+
for (const sub of unfiled) {
|
|
59
|
+
sections.push(feedToOutlineXml(sub, " "));
|
|
60
|
+
}
|
|
61
|
+
const outlines = sections.join("\n");
|
|
62
|
+
const now = new Date().toUTCString();
|
|
63
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
64
|
+
<opml version="2.0">
|
|
65
|
+
<head>
|
|
66
|
+
<title>FeedKeeper Subscriptions</title>
|
|
67
|
+
<dateCreated>${now}</dateCreated>
|
|
68
|
+
</head>
|
|
69
|
+
<body>
|
|
70
|
+
${outlines}
|
|
71
|
+
</body>
|
|
72
|
+
</opml>
|
|
73
|
+
`;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Recursively extracts outline elements containing an xmlUrl attribute from parsed OPML.
|
|
77
|
+
*/
|
|
78
|
+
function extractOutlines(node, items = [], currentFolder) {
|
|
79
|
+
if (!node || typeof node !== "object")
|
|
80
|
+
return items;
|
|
81
|
+
if (Array.isArray(node)) {
|
|
82
|
+
for (const item of node) {
|
|
83
|
+
extractOutlines(item, items, currentFolder);
|
|
84
|
+
}
|
|
85
|
+
return items;
|
|
86
|
+
}
|
|
87
|
+
const record = node;
|
|
88
|
+
const xmlUrl = (typeof record["@_xmlUrl"] === "string" ? record["@_xmlUrl"] : undefined) ??
|
|
89
|
+
(typeof record["@_xmlurl"] === "string" ? record["@_xmlurl"] : undefined);
|
|
90
|
+
const rawLabel = (typeof record["@_text"] === "string" ? record["@_text"] : undefined) ??
|
|
91
|
+
(typeof record["@_title"] === "string" ? record["@_title"] : undefined);
|
|
92
|
+
if (xmlUrl) {
|
|
93
|
+
const label = rawLabel?.trim() ? decodeEntities(rawLabel.trim()) : undefined;
|
|
94
|
+
items.push({ url: xmlUrl.trim(), label, folder: currentFolder });
|
|
95
|
+
}
|
|
96
|
+
else if (rawLabel && record.outline) {
|
|
97
|
+
currentFolder = decodeEntities(rawLabel.trim());
|
|
98
|
+
}
|
|
99
|
+
if (record.outline) {
|
|
100
|
+
extractOutlines(record.outline, items, currentFolder);
|
|
101
|
+
}
|
|
102
|
+
return items;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Parses an OPML document string and returns a list of feed URLs and their labels.
|
|
106
|
+
*/
|
|
107
|
+
export function parseOpml(xmlContent) {
|
|
108
|
+
const parser = new XMLParser({
|
|
109
|
+
ignoreAttributes: false,
|
|
110
|
+
attributeNamePrefix: "@_",
|
|
111
|
+
trimValues: true,
|
|
112
|
+
});
|
|
113
|
+
const parsed = parser.parse(xmlContent);
|
|
114
|
+
if (!parsed || !parsed.opml || !parsed.opml.body) {
|
|
115
|
+
throw new Error("invalid_opml_format");
|
|
116
|
+
}
|
|
117
|
+
return extractOutlines(parsed.opml.body.outline);
|
|
118
|
+
}
|
|
119
|
+
const urlSchema = z.string().url();
|
|
120
|
+
/**
|
|
121
|
+
* Imports feeds from an OPML document string for the given user.
|
|
122
|
+
*/
|
|
123
|
+
export async function importOpmlFeeds(userId, xmlContent) {
|
|
124
|
+
const feedItems = parseOpml(xmlContent);
|
|
125
|
+
const result = {
|
|
126
|
+
imported: 0,
|
|
127
|
+
skipped: 0,
|
|
128
|
+
failed: 0,
|
|
129
|
+
errors: [],
|
|
130
|
+
};
|
|
131
|
+
const feedsToPoll = [];
|
|
132
|
+
const folderCache = new Map();
|
|
133
|
+
for (const item of feedItems) {
|
|
134
|
+
if (!item.url)
|
|
135
|
+
continue;
|
|
136
|
+
const urlCheck = urlSchema.safeParse(item.url);
|
|
137
|
+
if (!urlCheck.success) {
|
|
138
|
+
result.failed++;
|
|
139
|
+
result.errors.push({ url: item.url, reason: "invalid_url_format" });
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
let normalizedUrl;
|
|
143
|
+
try {
|
|
144
|
+
const validated = await assertPublicHttpUrl(item.url);
|
|
145
|
+
normalizedUrl = validated.toString();
|
|
146
|
+
}
|
|
147
|
+
catch (error) {
|
|
148
|
+
result.failed++;
|
|
149
|
+
const reason = error instanceof SsrfBlockedError ? error.message : "url_validation_failed";
|
|
150
|
+
result.errors.push({ url: item.url, reason });
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
try {
|
|
154
|
+
let feed = findFeedByUrl(normalizedUrl);
|
|
155
|
+
if (!feed) {
|
|
156
|
+
feed = createFeed(normalizedUrl, Math.max(config.minPollIntervalMinutes, 15));
|
|
157
|
+
}
|
|
158
|
+
if (isUserSubscribed(userId, feed.id)) {
|
|
159
|
+
result.skipped++;
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
let folderId = null;
|
|
163
|
+
if (item.folder) {
|
|
164
|
+
if (folderCache.has(item.folder)) {
|
|
165
|
+
folderId = folderCache.get(item.folder);
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
const folder = createFolder(userId, item.folder);
|
|
169
|
+
folderId = folder.id;
|
|
170
|
+
folderCache.set(item.folder, folder.id);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
subscribe(userId, feed.id, item.label ?? null, folderId);
|
|
174
|
+
result.imported++;
|
|
175
|
+
feedsToPoll.push(feed);
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
result.failed++;
|
|
179
|
+
result.errors.push({
|
|
180
|
+
url: item.url,
|
|
181
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
// Durable admission only: no detached serial loop or Promise per imported feed.
|
|
186
|
+
for (const feed of feedsToPoll)
|
|
187
|
+
requestFeedPoll(feed);
|
|
188
|
+
return result;
|
|
189
|
+
}
|
|
190
|
+
//# sourceMappingURL=opml.js.map
|