@rackbops/docket-types 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -2
- package/dist/index.d.ts +13 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +29 -4
- package/dist/index.js.map +1 -1
- package/dist/scout.d.ts +127 -0
- package/dist/scout.d.ts.map +1 -0
- package/dist/scout.js +313 -0
- package/dist/scout.js.map +1 -0
- package/dist/want-bgg.d.ts +40 -0
- package/dist/want-bgg.d.ts.map +1 -0
- package/dist/want-bgg.js +196 -0
- package/dist/want-bgg.js.map +1 -0
- package/dist/want-sources.d.ts +67 -0
- package/dist/want-sources.d.ts.map +1 -0
- package/dist/want-sources.js +196 -0
- package/dist/want-sources.js.map +1 -0
- package/dist/wantjudge.d.ts +111 -0
- package/dist/wantjudge.d.ts.map +1 -0
- package/dist/wantjudge.js +437 -0
- package/dist/wantjudge.js.map +1 -0
- package/dist/wantlist.d.ts +73 -0
- package/dist/wantlist.d.ts.map +1 -0
- package/dist/wantlist.js +185 -0
- package/dist/wantlist.js.map +1 -0
- package/package.json +3 -3
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { type Fetch } from "@rackbops/docket-core";
|
|
2
|
+
import { type Listing, type Source } from "./want-sources.js";
|
|
3
|
+
/**
|
|
4
|
+
* The BoardGameGeek source (rackbops-bot-plugins#83; the #83a plan of 2026-09-29): a board game's
|
|
5
|
+
* marketplace listings from BGG's XML API, `GET https://boardgamegeek.com/xmlapi2/thing?id=<id>&marketplace=1`
|
|
6
|
+
* with `Authorization: Bearer <token>` (the host's; the tracker plugin's `TRACKER_BGG_TOKEN`) -- BGG's rule for registered applications
|
|
7
|
+
* (boardgamegeek.com/using_the_xml_api, read 2026-09-29) -- on the bare host (never `www.`), at
|
|
8
|
+
* least `BGG_SPACING_MS` between two requests. A host registers the source only once it has a
|
|
9
|
+
* token. While none has been issued (roshne applied on 2026-09-29),
|
|
10
|
+
* **the parser's fixture is hand-written to the documented shape, not a captured response**:
|
|
11
|
+
* one real response, saved once the token exists, is the check.
|
|
12
|
+
*
|
|
13
|
+
* The XML is read with `indexOf` and patterns over one bounded tag at a time, never over the whole
|
|
14
|
+
* body, so a large or broken answer cannot take the bot's thread for long; no XML library.
|
|
15
|
+
*/
|
|
16
|
+
export declare const BGG_HOST = "boardgamegeek.com";
|
|
17
|
+
export declare const BGG_SPACING_MS = 5000;
|
|
18
|
+
export declare const BGG_ATTRIBUTION = "via BoardGameGeek";
|
|
19
|
+
/** A BGG thing id from a bare number or a `boardgamegeek.com/boardgame/<id>/...` address, or null. */
|
|
20
|
+
export declare function parseBggThingId(input: string): number | null;
|
|
21
|
+
export declare function bggThingUrl(id: number): string;
|
|
22
|
+
/** `<listing>` elements looked at in one answer, kept or not: the bound on the walk. */
|
|
23
|
+
export declare const MAX_SCANNED = 1000;
|
|
24
|
+
/**
|
|
25
|
+
* The marketplace listings in a `thing` answer: each `<listing>` inside `<marketplacelistings>`,
|
|
26
|
+
* with its `price` (`currency`, `value`), `condition` and `link` (`href`), titled by the thing's
|
|
27
|
+
* primary `<name>`; a seller's `notes` are not read (free text, never put in a DM). A listing without
|
|
28
|
+
* a usable link is dropped. Each listing's tags are looked for only inside that listing's own text,
|
|
29
|
+
* and at most `MAX_SCANNED` listings are looked at, so the walk is linear in the answer. Pure.
|
|
30
|
+
*/
|
|
31
|
+
export declare function parseMarketplace(xml: string): Listing[];
|
|
32
|
+
export interface BggOptions {
|
|
33
|
+
token: string;
|
|
34
|
+
/** Raw bodies with no redirects followed (the body as sent, no redirect followed; the tracker plugin's `raw` and `noRedirects` page fetch): the XML as sent, fenced like every other read, and the token never leaves BGG's host. */
|
|
35
|
+
fetch: Fetch;
|
|
36
|
+
now?: () => number;
|
|
37
|
+
}
|
|
38
|
+
/** The `bgg` source: one request at a time, at least `BGG_SPACING_MS` apart, with the token. */
|
|
39
|
+
export declare function bggSource(o: BggOptions): Source;
|
|
40
|
+
//# sourceMappingURL=want-bgg.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"want-bgg.d.ts","sourceRoot":"","sources":["../src/want-bgg.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmC,KAAK,KAAK,EAAE,MAAM,uBAAuB,CAAA;AAGnF,OAAO,EACL,KAAK,OAAO,EAEZ,KAAK,MAAM,EAGZ,MAAM,mBAAmB,CAAA;AAE1B;;;;;;;;;;;;GAYG;AAEH,eAAO,MAAM,QAAQ,sBAAsB,CAAA;AAC3C,eAAO,MAAM,cAAc,OAAQ,CAAA;AACnC,eAAO,MAAM,eAAe,sBAAsB,CAAA;AASlD,sGAAsG;AACtG,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAe5D;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAE9C;AAoDD,wFAAwF;AACxF,eAAO,MAAM,WAAW,OAAQ,CAAA;AAEhC;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,EAAE,CAkCvD;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,qOAAqO;IACrO,KAAK,EAAE,KAAK,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CACnB;AAED,gGAAgG;AAChG,wBAAgB,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,MAAM,CA+C/C"}
|
package/dist/want-bgg.js
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { clean, ExecutorUnavailableError } from "@rackbops/docket-core";
|
|
2
|
+
import { parsePrice } from "./extract.js";
|
|
3
|
+
import { safeUrl } from "./research-answer.js";
|
|
4
|
+
import { MAX_LISTINGS, SourceMiss, SourceUnavailableError, } from "./want-sources.js";
|
|
5
|
+
/**
|
|
6
|
+
* The BoardGameGeek source (rackbops-bot-plugins#83; the #83a plan of 2026-09-29): a board game's
|
|
7
|
+
* marketplace listings from BGG's XML API, `GET https://boardgamegeek.com/xmlapi2/thing?id=<id>&marketplace=1`
|
|
8
|
+
* with `Authorization: Bearer <token>` (the host's; the tracker plugin's `TRACKER_BGG_TOKEN`) -- BGG's rule for registered applications
|
|
9
|
+
* (boardgamegeek.com/using_the_xml_api, read 2026-09-29) -- on the bare host (never `www.`), at
|
|
10
|
+
* least `BGG_SPACING_MS` between two requests. A host registers the source only once it has a
|
|
11
|
+
* token. While none has been issued (roshne applied on 2026-09-29),
|
|
12
|
+
* **the parser's fixture is hand-written to the documented shape, not a captured response**:
|
|
13
|
+
* one real response, saved once the token exists, is the check.
|
|
14
|
+
*
|
|
15
|
+
* The XML is read with `indexOf` and patterns over one bounded tag at a time, never over the whole
|
|
16
|
+
* body, so a large or broken answer cannot take the bot's thread for long; no XML library.
|
|
17
|
+
*/
|
|
18
|
+
export const BGG_HOST = "boardgamegeek.com";
|
|
19
|
+
export const BGG_SPACING_MS = 5_000;
|
|
20
|
+
export const BGG_ATTRIBUTION = "via BoardGameGeek";
|
|
21
|
+
/*
|
|
22
|
+
* A BGG read that would have to wait for the spacing is put back for the next tick instead
|
|
23
|
+
* (docket's `ExecutorUnavailableError` requeues a notify-lane run; it is not a miss): docket runs a
|
|
24
|
+
* tick's due runs one after another, so a wait here would hold the poll tick -- and the price checks
|
|
25
|
+
* behind it -- for 5 s a watch. So at most one BGG read starts per `BGG_SPACING_MS`.
|
|
26
|
+
*/
|
|
27
|
+
const MAX_TAG = 2_000;
|
|
28
|
+
/** A BGG thing id from a bare number or a `boardgamegeek.com/boardgame/<id>/...` address, or null. */
|
|
29
|
+
export function parseBggThingId(input) {
|
|
30
|
+
const raw = input.trim();
|
|
31
|
+
if (/^[0-9]{1,9}$/.test(raw))
|
|
32
|
+
return Number(raw) > 0 ? Number(raw) : null;
|
|
33
|
+
let url;
|
|
34
|
+
try {
|
|
35
|
+
url = new URL(raw);
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
if (url.protocol !== "https:" && url.protocol !== "http:")
|
|
41
|
+
return null;
|
|
42
|
+
if (url.hostname !== BGG_HOST && url.hostname !== `www.${BGG_HOST}`)
|
|
43
|
+
return null;
|
|
44
|
+
const m = /^\/(boardgame|boardgameexpansion|boardgameaccessory|thing)\/([0-9]{1,9})(\/|$)/.exec(url.pathname);
|
|
45
|
+
return m && Number(m[2]) > 0 ? Number(m[2]) : null;
|
|
46
|
+
}
|
|
47
|
+
export function bggThingUrl(id) {
|
|
48
|
+
return `https://${BGG_HOST}/xmlapi2/thing?id=${id}&marketplace=1`;
|
|
49
|
+
}
|
|
50
|
+
function unescapeXml(s) {
|
|
51
|
+
return s.replace(/&(#x[0-9a-f]{1,6}|#[0-9]{1,7}|amp|lt|gt|quot|apos);/gi, (_m, e) => {
|
|
52
|
+
const lower = e.toLowerCase();
|
|
53
|
+
if (lower === "amp")
|
|
54
|
+
return "&";
|
|
55
|
+
if (lower === "lt")
|
|
56
|
+
return "<";
|
|
57
|
+
if (lower === "gt")
|
|
58
|
+
return ">";
|
|
59
|
+
if (lower === "quot")
|
|
60
|
+
return '"';
|
|
61
|
+
if (lower === "apos")
|
|
62
|
+
return "'";
|
|
63
|
+
const code = lower.startsWith("#x")
|
|
64
|
+
? Number.parseInt(lower.slice(2), 16)
|
|
65
|
+
: Number.parseInt(lower.slice(1), 10);
|
|
66
|
+
return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : "";
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
/** The attributes of each `<name ...>` tag in `xml` (one listing's text, or the head before the listings), in order. */
|
|
70
|
+
function tagsAttrs(xml, name, limit = 50) {
|
|
71
|
+
const out = [];
|
|
72
|
+
let at = 0;
|
|
73
|
+
while (out.length < limit) {
|
|
74
|
+
const start = xml.indexOf(`<${name}`, at);
|
|
75
|
+
if (start < 0)
|
|
76
|
+
break;
|
|
77
|
+
at = start + name.length + 1;
|
|
78
|
+
const after = xml.charAt(at);
|
|
79
|
+
if (after !== " " &&
|
|
80
|
+
after !== "/" &&
|
|
81
|
+
after !== ">" &&
|
|
82
|
+
after !== "\t" &&
|
|
83
|
+
after !== "\n" &&
|
|
84
|
+
after !== "\r")
|
|
85
|
+
continue;
|
|
86
|
+
const end = xml.indexOf(">", at);
|
|
87
|
+
if (end < 0 || end - start > MAX_TAG)
|
|
88
|
+
break;
|
|
89
|
+
const attrs = {};
|
|
90
|
+
for (const m of xml
|
|
91
|
+
.slice(start, end)
|
|
92
|
+
.matchAll(/([A-Za-z_:][-A-Za-z0-9_:.]*)\s*=\s*"([^"]*)"/g)) {
|
|
93
|
+
attrs[(m[1] ?? "").toLowerCase()] = unescapeXml(m[2] ?? "");
|
|
94
|
+
}
|
|
95
|
+
out.push(attrs);
|
|
96
|
+
at = end + 1;
|
|
97
|
+
}
|
|
98
|
+
return out;
|
|
99
|
+
}
|
|
100
|
+
const first = (xml, name) => tagsAttrs(xml, name, 1)[0];
|
|
101
|
+
/** `<listing>` elements looked at in one answer, kept or not: the bound on the walk. */
|
|
102
|
+
export const MAX_SCANNED = 1_000;
|
|
103
|
+
/**
|
|
104
|
+
* The marketplace listings in a `thing` answer: each `<listing>` inside `<marketplacelistings>`,
|
|
105
|
+
* with its `price` (`currency`, `value`), `condition` and `link` (`href`), titled by the thing's
|
|
106
|
+
* primary `<name>`; a seller's `notes` are not read (free text, never put in a DM). A listing without
|
|
107
|
+
* a usable link is dropped. Each listing's tags are looked for only inside that listing's own text,
|
|
108
|
+
* and at most `MAX_SCANNED` listings are looked at, so the walk is linear in the answer. Pure.
|
|
109
|
+
*/
|
|
110
|
+
export function parseMarketplace(xml) {
|
|
111
|
+
const open = xml.indexOf("<marketplacelistings");
|
|
112
|
+
const head = open < 0 ? xml.slice(0, 200_000) : xml.slice(0, open);
|
|
113
|
+
const names = tagsAttrs(head, "name");
|
|
114
|
+
const primary = names.find((n) => n.type === "primary") ?? names[0];
|
|
115
|
+
const title = clean(primary?.value ?? "", 150, true) || "A BGG listing";
|
|
116
|
+
const out = [];
|
|
117
|
+
if (open < 0)
|
|
118
|
+
return out;
|
|
119
|
+
const close = xml.indexOf("</marketplacelistings>", open);
|
|
120
|
+
const stop = close < 0 ? xml.length : close;
|
|
121
|
+
let at = open;
|
|
122
|
+
for (let scanned = 0; scanned < MAX_SCANNED && out.length < MAX_LISTINGS; scanned++) {
|
|
123
|
+
const start = xml.indexOf("<listing>", at);
|
|
124
|
+
if (start < 0 || start >= stop)
|
|
125
|
+
break;
|
|
126
|
+
const endTag = xml.indexOf("</listing>", start);
|
|
127
|
+
const end = endTag < 0 || endTag > stop ? stop : endTag;
|
|
128
|
+
at = end + 1;
|
|
129
|
+
const one = xml.slice(start, end);
|
|
130
|
+
const url = safeUrl(first(one, "link")?.href ?? "");
|
|
131
|
+
if (!url)
|
|
132
|
+
continue;
|
|
133
|
+
const price = first(one, "price");
|
|
134
|
+
const value = price?.value !== undefined ? parsePrice(price.value) : null;
|
|
135
|
+
const currency = (price?.currency ?? "").trim().toUpperCase();
|
|
136
|
+
const condition = clean(first(one, "condition")?.value ?? "", 40, true);
|
|
137
|
+
out.push({
|
|
138
|
+
id: `bgg:${url}`,
|
|
139
|
+
title,
|
|
140
|
+
url,
|
|
141
|
+
...(value !== null ? { price: value } : {}),
|
|
142
|
+
...(/^[A-Z]{3}$/.test(currency) ? { currency } : {}),
|
|
143
|
+
...(condition ? { condition } : {}),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
return out;
|
|
147
|
+
}
|
|
148
|
+
/** The `bgg` source: one request at a time, at least `BGG_SPACING_MS` apart, with the token. */
|
|
149
|
+
export function bggSource(o) {
|
|
150
|
+
const now = o.now ?? Date.now;
|
|
151
|
+
let last = Number.NEGATIVE_INFINITY;
|
|
152
|
+
let busy = false;
|
|
153
|
+
const request = async (id) => {
|
|
154
|
+
last = now();
|
|
155
|
+
let status;
|
|
156
|
+
let body;
|
|
157
|
+
try {
|
|
158
|
+
const response = await o.fetch.get(bggThingUrl(id), {
|
|
159
|
+
authorization: `Bearer ${o.token}`,
|
|
160
|
+
accept: "application/xml,text/xml;q=0.9,*/*;q=0.5",
|
|
161
|
+
});
|
|
162
|
+
status = response.status;
|
|
163
|
+
body = response.body;
|
|
164
|
+
}
|
|
165
|
+
catch (err) {
|
|
166
|
+
throw new SourceMiss(`BGG could not be read: ${err instanceof Error ? err.message : String(err)}`);
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
last = now();
|
|
170
|
+
}
|
|
171
|
+
if (status === 401 || status === 403)
|
|
172
|
+
throw new SourceUnavailableError("BGG rejected the token");
|
|
173
|
+
if (status === 202 || status === 429 || status >= 500)
|
|
174
|
+
throw new SourceMiss(`BGG is busy (HTTP ${status})`);
|
|
175
|
+
// A redirect is not followed (the token goes to boardgamegeek.com only): it is a miss like any other answer.
|
|
176
|
+
if (status !== 200)
|
|
177
|
+
throw new SourceMiss(`BGG answered HTTP ${status}`);
|
|
178
|
+
return parseMarketplace(body);
|
|
179
|
+
};
|
|
180
|
+
return {
|
|
181
|
+
id: "bgg",
|
|
182
|
+
search(target) {
|
|
183
|
+
const id = parseBggThingId(target);
|
|
184
|
+
if (id === null)
|
|
185
|
+
return Promise.reject(new SourceMiss("that is not a BGG game"));
|
|
186
|
+
if (busy || now() < last + BGG_SPACING_MS) {
|
|
187
|
+
return Promise.reject(new ExecutorUnavailableError("BGG reads are spaced out; this one waits for the next tick"));
|
|
188
|
+
}
|
|
189
|
+
busy = true;
|
|
190
|
+
return request(id).finally(() => {
|
|
191
|
+
busy = false;
|
|
192
|
+
});
|
|
193
|
+
},
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=want-bgg.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"want-bgg.js","sourceRoot":"","sources":["../src/want-bgg.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAc,MAAM,uBAAuB,CAAA;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAA;AAC9C,OAAO,EAEL,YAAY,EAEZ,UAAU,EACV,sBAAsB,GACvB,MAAM,mBAAmB,CAAA;AAE1B;;;;;;;;;;;;GAYG;AAEH,MAAM,CAAC,MAAM,QAAQ,GAAG,mBAAmB,CAAA;AAC3C,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAA;AACnC,MAAM,CAAC,MAAM,eAAe,GAAG,mBAAmB,CAAA;AAClD;;;;;GAKG;AACH,MAAM,OAAO,GAAG,KAAK,CAAA;AAErB,sGAAsG;AACtG,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IACxB,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACzE,IAAI,GAAQ,CAAA;IACZ,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IACtE,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,QAAQ,EAAE;QAAE,OAAO,IAAI,CAAA;IAChF,MAAM,CAAC,GAAG,gFAAgF,CAAC,IAAI,CAC7F,GAAG,CAAC,QAAQ,CACb,CAAA;IACD,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;AACpD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,OAAO,WAAW,QAAQ,qBAAqB,EAAE,gBAAgB,CAAA;AACnE,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,uDAAuD,EAAE,CAAC,EAAE,EAAE,CAAS,EAAE,EAAE;QAC1F,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;QAC7B,IAAI,KAAK,KAAK,KAAK;YAAE,OAAO,GAAG,CAAA;QAC/B,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,GAAG,CAAA;QAC9B,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,GAAG,CAAA;QAC9B,IAAI,KAAK,KAAK,MAAM;YAAE,OAAO,GAAG,CAAA;QAChC,IAAI,KAAK,KAAK,MAAM;YAAE,OAAO,GAAG,CAAA;QAChC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;YACjC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACrC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACvC,OAAO,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACvE,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,wHAAwH;AACxH,SAAS,SAAS,CAAC,GAAW,EAAE,IAAY,EAAE,KAAK,GAAG,EAAE;IACtD,MAAM,GAAG,GAA6B,EAAE,CAAA;IACxC,IAAI,EAAE,GAAG,CAAC,CAAA;IACV,OAAO,GAAG,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,CAAC,CAAA;QACzC,IAAI,KAAK,GAAG,CAAC;YAAE,MAAK;QACpB,EAAE,GAAG,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC5B,IACE,KAAK,KAAK,GAAG;YACb,KAAK,KAAK,GAAG;YACb,KAAK,KAAK,GAAG;YACb,KAAK,KAAK,IAAI;YACd,KAAK,KAAK,IAAI;YACd,KAAK,KAAK,IAAI;YAEd,SAAQ;QACV,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QAChC,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,KAAK,GAAG,OAAO;YAAE,MAAK;QAC3C,MAAM,KAAK,GAA2B,EAAE,CAAA;QACxC,KAAK,MAAM,CAAC,IAAI,GAAG;aAChB,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;aACjB,QAAQ,CAAC,+CAA+C,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACf,EAAE,GAAG,GAAG,GAAG,CAAC,CAAA;IACd,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,IAAY,EAAsC,EAAE,CAC9E,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AAE5B,wFAAwF;AACxF,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,CAAA;AAEhC;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAA;IAChD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAClE,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACrC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAA;IACnE,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,eAAe,CAAA;IACvE,MAAM,GAAG,GAAc,EAAE,CAAA;IACzB,IAAI,IAAI,GAAG,CAAC;QAAE,OAAO,GAAG,CAAA;IACxB,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAA;IACzD,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAA;IAC3C,IAAI,EAAE,GAAG,IAAI,CAAA;IACb,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,IAAI,GAAG,CAAC,MAAM,GAAG,YAAY,EAAE,OAAO,EAAE,EAAE,CAAC;QACpF,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;QAC1C,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI;YAAE,MAAK;QACrC,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;QAC/C,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAA;QACvD,EAAE,GAAG,GAAG,GAAG,CAAC,CAAA;QACZ,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QACjC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAA;QACnD,IAAI,CAAC,GAAG;YAAE,SAAQ;QAClB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACjC,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QACzE,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAC7D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,KAAK,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;QACvE,GAAG,CAAC,IAAI,CAAC;YACP,EAAE,EAAE,OAAO,GAAG,EAAE;YAChB,KAAK;YACL,GAAG;YACH,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpC,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AASD,gGAAgG;AAChG,MAAM,UAAU,SAAS,CAAC,CAAa;IACrC,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAA;IAC7B,IAAI,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAA;IACnC,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,MAAM,OAAO,GAAG,KAAK,EAAE,EAAU,EAAsB,EAAE;QACvD,IAAI,GAAG,GAAG,EAAE,CAAA;QACZ,IAAI,MAAc,CAAA;QAClB,IAAI,IAAY,CAAA;QAChB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE;gBAClD,aAAa,EAAE,UAAU,CAAC,CAAC,KAAK,EAAE;gBAClC,MAAM,EAAE,0CAA0C;aACnD,CAAC,CAAA;YACF,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;YACxB,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAClB,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC7E,CAAA;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,GAAG,GAAG,EAAE,CAAA;QACd,CAAC;QACD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;YAAE,MAAM,IAAI,sBAAsB,CAAC,wBAAwB,CAAC,CAAA;QAChG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,IAAI,GAAG;YACnD,MAAM,IAAI,UAAU,CAAC,qBAAqB,MAAM,GAAG,CAAC,CAAA;QACtD,6GAA6G;QAC7G,IAAI,MAAM,KAAK,GAAG;YAAE,MAAM,IAAI,UAAU,CAAC,qBAAqB,MAAM,EAAE,CAAC,CAAA;QACvE,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAC/B,CAAC,CAAA;IACD,OAAO;QACL,EAAE,EAAE,KAAK;QACT,MAAM,CAAC,MAAM;YACX,MAAM,EAAE,GAAG,eAAe,CAAC,MAAM,CAAC,CAAA;YAClC,IAAI,EAAE,KAAK,IAAI;gBAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,wBAAwB,CAAC,CAAC,CAAA;YAChF,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,GAAG,cAAc,EAAE,CAAC;gBAC1C,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,wBAAwB,CAC1B,4DAA4D,CAC7D,CACF,CAAA;YACH,CAAC;YACD,IAAI,GAAG,IAAI,CAAA;YACX,OAAO,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;gBAC9B,IAAI,GAAG,KAAK,CAAA;YACd,CAAC,CAAC,CAAA;QACJ,CAAC;KACF,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { type Fetch } from "@rackbops/docket-core";
|
|
2
|
+
/**
|
|
3
|
+
* The want-list watcher's sources (category 2, plan 1.2 row 2; rackbops-bot-plugins#83): where a
|
|
4
|
+
* wanted thing's listings come from, behind one port so the type (wantlist.ts) knows none of
|
|
5
|
+
* them. Two read listings:
|
|
6
|
+
*
|
|
7
|
+
* - `page` (here): a listing or search page the owner pasted (roshne, 2026-10-04, "Pages too"), read
|
|
8
|
+
* through the fenced Fetch port as `/price` reads a product page, and only its structured data --
|
|
9
|
+
* JSON-LD `ItemList`s of products and standalone `Product`s. No guessing at HTML: a page without
|
|
10
|
+
* them yields nothing, and the owner is told.
|
|
11
|
+
* - `bgg` (want-bgg.ts): BoardGameGeek's marketplace, through its XML API, off until BGG approves
|
|
12
|
+
* roshne's application and `TRACKER_BGG_TOKEN` is set.
|
|
13
|
+
*
|
|
14
|
+
* eBay is a third answer, not a source: the tracker never reads eBay (no API, no pages); `/want`
|
|
15
|
+
* hands the owner an eBay search to save on eBay, whose own alerts do the watching (the tracker plugin's `/want`).
|
|
16
|
+
*/
|
|
17
|
+
export interface Listing {
|
|
18
|
+
/** Stable for one listing: its URL without the fragment, for `page`; `bgg:<link>` for BGG. */
|
|
19
|
+
id: string;
|
|
20
|
+
title: string;
|
|
21
|
+
url: string;
|
|
22
|
+
price?: number;
|
|
23
|
+
currency?: string;
|
|
24
|
+
condition?: string;
|
|
25
|
+
seller?: string;
|
|
26
|
+
}
|
|
27
|
+
export type SourceId = "page" | "bgg";
|
|
28
|
+
export declare const SOURCE_IDS: readonly SourceId[];
|
|
29
|
+
export interface Source {
|
|
30
|
+
id: SourceId;
|
|
31
|
+
/** The listings at `target` now; throws `SourceMiss` for a read that counts as a miss. */
|
|
32
|
+
search(target: string, fetch: Fetch | undefined): Promise<Listing[]>;
|
|
33
|
+
}
|
|
34
|
+
/** A read that found nothing usable (an error page, no structured data, a busy API): the type counts it. */
|
|
35
|
+
export declare class SourceMiss extends Error {
|
|
36
|
+
name: string;
|
|
37
|
+
}
|
|
38
|
+
/** The source cannot be read at all as set up (BGG refusing the token): a miss the owner should hear about. */
|
|
39
|
+
export declare class SourceUnavailableError extends SourceMiss {
|
|
40
|
+
name: string;
|
|
41
|
+
}
|
|
42
|
+
export declare const MAX_LISTINGS = 100;
|
|
43
|
+
/** The key a listing is remembered by: a digest of its id, so the state stays small whatever the URLs. */
|
|
44
|
+
export declare function listingKey(id: string): string;
|
|
45
|
+
/**
|
|
46
|
+
* A request header that is a directive to the host's Fetch port, never sent: any hop to eBay is
|
|
47
|
+
* refused. The page source's reads carry it, so a pasted page that redirects to eBay is not read
|
|
48
|
+
* either (roshne's rule: eBay's own saved-search alerts cover eBay; the tracker never reads it).
|
|
49
|
+
* A host's Fetch adapter must honour it and strip it.
|
|
50
|
+
*/
|
|
51
|
+
export declare const NEVER_EBAY = "x-tracker-never-ebay";
|
|
52
|
+
/** eBay's own hosts (and its short links). */
|
|
53
|
+
export declare function isEbayHost(host: string): boolean;
|
|
54
|
+
/**
|
|
55
|
+
* The listings in a page's JSON-LD: each `ItemList`'s elements (a `ListItem`'s `item`, or the
|
|
56
|
+
* element itself) and each standalone product, in page order, without repeats. A product with no
|
|
57
|
+
* address of its own is the page itself (a product page). Pure.
|
|
58
|
+
*/
|
|
59
|
+
export declare function listingsFromJsonLd(html: string, pageUrl: string): Listing[];
|
|
60
|
+
/** `listingsFromJsonLd`, and how many `ItemList`s the page declares: an empty one is a search with no results, not an unreadable page. */
|
|
61
|
+
export declare function readListings(html: string, pageUrl: string): {
|
|
62
|
+
listings: Listing[];
|
|
63
|
+
lists: number;
|
|
64
|
+
};
|
|
65
|
+
/** The `page` source: the owner's page through the fenced Fetch port, structured data only. */
|
|
66
|
+
export declare const pageSource: Source;
|
|
67
|
+
//# sourceMappingURL=want-sources.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"want-sources.d.ts","sourceRoot":"","sources":["../src/want-sources.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,KAAK,EAAE,MAAM,uBAAuB,CAAA;AAIzD;;;;;;;;;;;;;;GAcG;AAEH,MAAM,WAAW,OAAO;IACtB,8FAA8F;IAC9F,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAA;AACrC,eAAO,MAAM,UAAU,EAAE,SAAS,QAAQ,EAAoB,CAAA;AAE9D,MAAM,WAAW,MAAM;IACrB,EAAE,EAAE,QAAQ,CAAA;IACZ,0FAA0F;IAC1F,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;CACrE;AAED,4GAA4G;AAC5G,qBAAa,UAAW,SAAQ,KAAK;IAC1B,IAAI,SAAe;CAC7B;AAED,+GAA+G;AAC/G,qBAAa,sBAAuB,SAAQ,UAAU;IAC3C,IAAI,SAA2B;CACzC;AAED,eAAO,MAAM,YAAY,MAAM,CAAA;AAM/B,0GAA0G;AAC1G,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAE7C;AAED;;;;;GAKG;AACH,eAAO,MAAM,UAAU,yBAAyB,CAAA;AAEhD,8CAA8C;AAC9C,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEhD;AAkFD;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,EAAE,CAE3E;AAED,0IAA0I;AAC1I,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,GACd;IAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAuCxC;AAED,+FAA+F;AAC/F,eAAO,MAAM,UAAU,EAAE,MAsBxB,CAAA"}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { clean } from "@rackbops/docket-core";
|
|
3
|
+
import { jsonLdBlocks, priceFromJson } from "./extract.js";
|
|
4
|
+
import { safeUrl } from "./research-answer.js";
|
|
5
|
+
export const SOURCE_IDS = ["page", "bgg"];
|
|
6
|
+
/** A read that found nothing usable (an error page, no structured data, a busy API): the type counts it. */
|
|
7
|
+
export class SourceMiss extends Error {
|
|
8
|
+
name = "SourceMiss";
|
|
9
|
+
}
|
|
10
|
+
/** The source cannot be read at all as set up (BGG refusing the token): a miss the owner should hear about. */
|
|
11
|
+
export class SourceUnavailableError extends SourceMiss {
|
|
12
|
+
name = "SourceUnavailableError";
|
|
13
|
+
}
|
|
14
|
+
export const MAX_LISTINGS = 100;
|
|
15
|
+
const TITLE_CHARS = 150;
|
|
16
|
+
const CONDITION_CHARS = 40;
|
|
17
|
+
const SELLER_CHARS = 80;
|
|
18
|
+
const MAX_DEPTH = 8;
|
|
19
|
+
/** The key a listing is remembered by: a digest of its id, so the state stays small whatever the URLs. */
|
|
20
|
+
export function listingKey(id) {
|
|
21
|
+
return createHash("sha256").update(id).digest("hex").slice(0, 32);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* A request header that is a directive to the host's Fetch port, never sent: any hop to eBay is
|
|
25
|
+
* refused. The page source's reads carry it, so a pasted page that redirects to eBay is not read
|
|
26
|
+
* either (roshne's rule: eBay's own saved-search alerts cover eBay; the tracker never reads it).
|
|
27
|
+
* A host's Fetch adapter must honour it and strip it.
|
|
28
|
+
*/
|
|
29
|
+
export const NEVER_EBAY = "x-tracker-never-ebay";
|
|
30
|
+
/** eBay's own hosts (and its short links). */
|
|
31
|
+
export function isEbayHost(host) {
|
|
32
|
+
return /(^|\.)ebay\.[a-z]{2,3}(\.[a-z]{2})?$/i.test(host.replace(/\.$/, ""));
|
|
33
|
+
}
|
|
34
|
+
const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
35
|
+
function typesOf(node) {
|
|
36
|
+
const t = node["@type"];
|
|
37
|
+
const all = Array.isArray(t) ? t : [t];
|
|
38
|
+
return all
|
|
39
|
+
.filter((x) => typeof x === "string")
|
|
40
|
+
.map((x) => x.replace(/^https?:\/\/schema\.org\//, ""));
|
|
41
|
+
}
|
|
42
|
+
const PRODUCT_TYPES = new Set([
|
|
43
|
+
"Product",
|
|
44
|
+
"IndividualProduct",
|
|
45
|
+
"ProductModel",
|
|
46
|
+
"ProductGroup",
|
|
47
|
+
"Book",
|
|
48
|
+
"Game",
|
|
49
|
+
"VideoGame",
|
|
50
|
+
"Vehicle",
|
|
51
|
+
]);
|
|
52
|
+
function text(v, max) {
|
|
53
|
+
if (typeof v === "string")
|
|
54
|
+
return clean(v, max, true);
|
|
55
|
+
if (isObj(v) && typeof v.name === "string")
|
|
56
|
+
return clean(v.name, max, true);
|
|
57
|
+
return "";
|
|
58
|
+
}
|
|
59
|
+
/** schema.org's condition as a word: `https://schema.org/UsedCondition` is "used". */
|
|
60
|
+
function conditionOf(v) {
|
|
61
|
+
const raw = text(v, 200);
|
|
62
|
+
const m = /(New|Used|Refurbished|Damaged)Condition$/i.exec(raw);
|
|
63
|
+
return m ? (m[1] ?? "").toLowerCase() : clean(raw, CONDITION_CHARS, true);
|
|
64
|
+
}
|
|
65
|
+
function firstOffer(offers) {
|
|
66
|
+
if (Array.isArray(offers))
|
|
67
|
+
return offers.find(isObj) ?? null;
|
|
68
|
+
return isObj(offers) ? offers : null;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Query parameters a shop adds per view or per search (Shopify's `_pos`/`_sid`/`_ss`, Google's
|
|
72
|
+
* `srsltid`, the `utm_*` family and the click ids): dropped from a listing's address, so one listing
|
|
73
|
+
* keeps one id from poll to poll and is not sent again as new.
|
|
74
|
+
*/
|
|
75
|
+
const PER_VIEW = /^(utm_[a-z]+|_pos|_sid|_ss|_psq|srsltid|gclid|gbraid|wbraid|fbclid|msclkid|mc_cid|mc_eid|pf_rd_[a-z]+|pd_rd_[a-z]+)$/i;
|
|
76
|
+
function absolute(raw, base) {
|
|
77
|
+
if (typeof raw !== "string" || raw.trim() === "")
|
|
78
|
+
return null;
|
|
79
|
+
try {
|
|
80
|
+
const u = new URL(raw.trim(), base);
|
|
81
|
+
u.hash = "";
|
|
82
|
+
for (const key of [...u.searchParams.keys()])
|
|
83
|
+
if (PER_VIEW.test(key))
|
|
84
|
+
u.searchParams.delete(key);
|
|
85
|
+
return safeUrl(u.toString());
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/** One listing from a product node, or null without a name or a usable address. */
|
|
92
|
+
function listingOf(node, base, fallbackUrl) {
|
|
93
|
+
const title = text(node.name, TITLE_CHARS);
|
|
94
|
+
const offer = firstOffer(node.offers);
|
|
95
|
+
const url = absolute(node.url, base) ?? absolute(fallbackUrl, base) ?? absolute(offer?.url, base);
|
|
96
|
+
if (!title || !url)
|
|
97
|
+
return null;
|
|
98
|
+
const price = priceFromJson(node.offers ?? null);
|
|
99
|
+
const condition = conditionOf(offer?.itemCondition ?? node.itemCondition);
|
|
100
|
+
const seller = text(offer?.seller, SELLER_CHARS);
|
|
101
|
+
return {
|
|
102
|
+
id: url,
|
|
103
|
+
title,
|
|
104
|
+
url,
|
|
105
|
+
...(price ? { price: price.value } : {}),
|
|
106
|
+
...(price?.currency ? { currency: price.currency } : {}),
|
|
107
|
+
...(condition ? { condition } : {}),
|
|
108
|
+
...(seller ? { seller } : {}),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* The listings in a page's JSON-LD: each `ItemList`'s elements (a `ListItem`'s `item`, or the
|
|
113
|
+
* element itself) and each standalone product, in page order, without repeats. A product with no
|
|
114
|
+
* address of its own is the page itself (a product page). Pure.
|
|
115
|
+
*/
|
|
116
|
+
export function listingsFromJsonLd(html, pageUrl) {
|
|
117
|
+
return readListings(html, pageUrl).listings;
|
|
118
|
+
}
|
|
119
|
+
/** `listingsFromJsonLd`, and how many `ItemList`s the page declares: an empty one is a search with no results, not an unreadable page. */
|
|
120
|
+
export function readListings(html, pageUrl) {
|
|
121
|
+
let lists = 0;
|
|
122
|
+
const out = [];
|
|
123
|
+
const seen = new Set();
|
|
124
|
+
const add = (l) => {
|
|
125
|
+
if (!l || seen.has(l.id) || out.length >= MAX_LISTINGS)
|
|
126
|
+
return;
|
|
127
|
+
seen.add(l.id);
|
|
128
|
+
out.push(l);
|
|
129
|
+
};
|
|
130
|
+
const walk = (node, depth) => {
|
|
131
|
+
if (depth > MAX_DEPTH || out.length >= MAX_LISTINGS)
|
|
132
|
+
return;
|
|
133
|
+
if (Array.isArray(node)) {
|
|
134
|
+
for (const n of node)
|
|
135
|
+
walk(n, depth + 1);
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
if (!isObj(node))
|
|
139
|
+
return;
|
|
140
|
+
const types = typesOf(node);
|
|
141
|
+
if (types.includes("ItemList")) {
|
|
142
|
+
lists++;
|
|
143
|
+
const elements = Array.isArray(node.itemListElement)
|
|
144
|
+
? node.itemListElement
|
|
145
|
+
: [node.itemListElement];
|
|
146
|
+
for (const el of elements) {
|
|
147
|
+
if (!isObj(el))
|
|
148
|
+
continue;
|
|
149
|
+
const item = isObj(el.item) ? el.item : el;
|
|
150
|
+
if (typesOf(item).includes("ItemList"))
|
|
151
|
+
walk(item, depth + 1);
|
|
152
|
+
else
|
|
153
|
+
add(listingOf(item, pageUrl, el.url ?? (typeof el.item === "string" ? el.item : undefined)));
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
else if (types.some((t) => PRODUCT_TYPES.has(t))) {
|
|
157
|
+
add(listingOf(node, pageUrl, pageUrl));
|
|
158
|
+
}
|
|
159
|
+
if (node["@graph"] !== undefined)
|
|
160
|
+
walk(node["@graph"], depth + 1);
|
|
161
|
+
if (isObj(node.mainEntity))
|
|
162
|
+
walk(node.mainEntity, depth + 1);
|
|
163
|
+
};
|
|
164
|
+
for (const block of jsonLdBlocks(html))
|
|
165
|
+
walk(block, 0);
|
|
166
|
+
return { listings: out, lists };
|
|
167
|
+
}
|
|
168
|
+
/** The `page` source: the owner's page through the fenced Fetch port, structured data only. */
|
|
169
|
+
export const pageSource = {
|
|
170
|
+
id: "page",
|
|
171
|
+
async search(target, fetch) {
|
|
172
|
+
if (!fetch)
|
|
173
|
+
throw new SourceMiss("this bot reads no pages");
|
|
174
|
+
let body;
|
|
175
|
+
try {
|
|
176
|
+
const response = await fetch.get(target, { [NEVER_EBAY]: "1" });
|
|
177
|
+
if (response.status !== 200)
|
|
178
|
+
throw new SourceMiss(`HTTP ${response.status}`);
|
|
179
|
+
body = response.body;
|
|
180
|
+
}
|
|
181
|
+
catch (err) {
|
|
182
|
+
if (err instanceof SourceMiss)
|
|
183
|
+
throw err;
|
|
184
|
+
// The cause is kept: `/want`'s first read tells a refused page from one that failed this once.
|
|
185
|
+
throw new SourceMiss(`the read failed: ${err instanceof Error ? err.message : String(err)}`, {
|
|
186
|
+
cause: err,
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
const { listings, lists } = readListings(body, target);
|
|
190
|
+
// A search with nothing for sale (the usual state of a want) still declares its list.
|
|
191
|
+
if (listings.length === 0 && lists === 0)
|
|
192
|
+
throw new SourceMiss("no listings in the page's structured data");
|
|
193
|
+
return listings;
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
//# sourceMappingURL=want-sources.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"want-sources.js","sourceRoot":"","sources":["../src/want-sources.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAc,MAAM,uBAAuB,CAAA;AACzD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAA;AA8B9C,MAAM,CAAC,MAAM,UAAU,GAAwB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;AAQ9D,4GAA4G;AAC5G,MAAM,OAAO,UAAW,SAAQ,KAAK;IAC1B,IAAI,GAAG,YAAY,CAAA;CAC7B;AAED,+GAA+G;AAC/G,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAC3C,IAAI,GAAG,wBAAwB,CAAA;CACzC;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,GAAG,CAAA;AAC/B,MAAM,WAAW,GAAG,GAAG,CAAA;AACvB,MAAM,eAAe,GAAG,EAAE,CAAA;AAC1B,MAAM,YAAY,GAAG,EAAE,CAAA;AACvB,MAAM,SAAS,GAAG,CAAC,CAAA;AAEnB,0GAA0G;AAC1G,MAAM,UAAU,UAAU,CAAC,EAAU;IACnC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;AACnE,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,sBAAsB,CAAA;AAEhD,8CAA8C;AAC9C,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,uCAAuC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAA;AAC9E,CAAC;AAGD,MAAM,KAAK,GAAG,CAAC,CAAU,EAAa,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAEjG,SAAS,OAAO,CAAC,IAAU;IACzB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAA;IACvB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACtC,OAAO,GAAG;SACP,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;SACjD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC,CAAA;AAC3D,CAAC;AAED,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,SAAS;IACT,mBAAmB;IACnB,cAAc;IACd,cAAc;IACd,MAAM;IACN,MAAM;IACN,WAAW;IACX,SAAS;CACV,CAAC,CAAA;AAEF,SAAS,IAAI,CAAC,CAAU,EAAE,GAAW;IACnC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;IACrD,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;IAC3E,OAAO,EAAE,CAAA;AACX,CAAC;AAED,sFAAsF;AACtF,SAAS,WAAW,CAAC,CAAU;IAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;IACxB,MAAM,CAAC,GAAG,2CAA2C,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC/D,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,eAAe,EAAE,IAAI,CAAC,CAAA;AAC3E,CAAC;AAED,SAAS,UAAU,CAAC,MAAe;IACjC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAA;IAC5D,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;AACtC,CAAC;AAED;;;;GAIG;AACH,MAAM,QAAQ,GACZ,uHAAuH,CAAA;AAEzH,SAAS,QAAQ,CAAC,GAAY,EAAE,IAAY;IAC1C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,IAAI,CAAA;IAC7D,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAA;QACnC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAA;QACX,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;YAAE,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAChG,OAAO,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,mFAAmF;AACnF,SAAS,SAAS,CAAC,IAAU,EAAE,IAAY,EAAE,WAAoB;IAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAC1C,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;IACjG,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAA;IAChD,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC,CAAA;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,CAAA;IAChD,OAAO;QACL,EAAE,EAAE,GAAG;QACP,KAAK;QACL,GAAG;QACH,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9B,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,OAAe;IAC9D,OAAO,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAA;AAC7C,CAAC;AAED,0IAA0I;AAC1I,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,OAAe;IAEf,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,MAAM,GAAG,GAAc,EAAE,CAAA;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,MAAM,GAAG,GAAG,CAAC,CAAiB,EAAE,EAAE;QAChC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,MAAM,IAAI,YAAY;YAAE,OAAM;QAC9D,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACb,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,CAAC,IAAa,EAAE,KAAa,EAAQ,EAAE;QAClD,IAAI,KAAK,GAAG,SAAS,IAAI,GAAG,CAAC,MAAM,IAAI,YAAY;YAAE,OAAM;QAC3D,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,IAAI;gBAAE,IAAI,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;YACxC,OAAM;QACR,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,OAAM;QACxB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAC3B,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,KAAK,EAAE,CAAA;YACP,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC;gBAClD,CAAC,CAAC,IAAI,CAAC,eAAe;gBACtB,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAC1B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBAAE,SAAQ;gBACxB,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAA;gBAC1C,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;oBAAE,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;;oBAE3D,GAAG,CACD,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CACxF,CAAA;YACL,CAAC;QACH,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAA;QACxC,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,SAAS;YAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;QACjE,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;IAC9D,CAAC,CAAA;IACD,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,IAAI,CAAC;QAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IACtD,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;AACjC,CAAC;AAED,+FAA+F;AAC/F,MAAM,CAAC,MAAM,UAAU,GAAW;IAChC,EAAE,EAAE,MAAM;IACV,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK;QACxB,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,UAAU,CAAC,yBAAyB,CAAC,CAAA;QAC3D,IAAI,IAAY,CAAA;QAChB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;YAC/D,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;gBAAE,MAAM,IAAI,UAAU,CAAC,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5E,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,UAAU;gBAAE,MAAM,GAAG,CAAA;YACxC,+FAA+F;YAC/F,MAAM,IAAI,UAAU,CAAC,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;gBAC3F,KAAK,EAAE,GAAG;aACX,CAAC,CAAA;QACJ,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QACtD,sFAAsF;QACtF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC;YACtC,MAAM,IAAI,UAAU,CAAC,2CAA2C,CAAC,CAAA;QACnE,OAAO,QAAQ,CAAA;IACjB,CAAC;CACF,CAAA"}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { type JobSpec, type TaskType } from "@rackbops/docket-core";
|
|
2
|
+
import { type Listing, type Source, type SourceId } from "./want-sources.js";
|
|
3
|
+
import { type WantConfig, type WantState } from "./wantlist.js";
|
|
4
|
+
/**
|
|
5
|
+
* The judged want-list watch (category 2, plan 5.4 and item 63; rackbops-bot-plugins#83): the
|
|
6
|
+
* watcher (wantlist.ts) with the model looking at each new listing before it is DMed -- is it
|
|
7
|
+
* the thing wanted, and what does the listing's own page show about the seller. "The model used only
|
|
8
|
+
* to judge new candidates" (5.4): the source is read in plain code, and a model Job is made only when
|
|
9
|
+
* that read finds listings not told before.
|
|
10
|
+
*
|
|
11
|
+
* A type of its own because a task's lane is its type's: on the execute lane, so a judged watch
|
|
12
|
+
* waits behind other model runs and stops while its owner is at a budget ceiling, and the plain
|
|
13
|
+
* watch keeps the notify lane's promptness. Two runs per batch, through the state:
|
|
14
|
+
*
|
|
15
|
+
* 1. the scheduled run's `prepare` reads the source; with nothing new it ends there, uncharged
|
|
16
|
+
* (`NoJob`); with new listings it keeps them as `pending` and asks for a follow-up;
|
|
17
|
+
* 2. that follow-up's `prepare` (or the next scheduled run's, if it comes first and no retry is
|
|
18
|
+
* waiting) turns `pending` into the judge Job, and `finish` maps its verdicts back onto
|
|
19
|
+
* `pending` by number. A follow-up never reads the source, so follow-ups never chain.
|
|
20
|
+
*
|
|
21
|
+
* `pending` lives in the state, not in memory, so a Job collected after a restart still has the
|
|
22
|
+
* listings it was about. The judge may open only listings on the watch's own site (`WebFetch`
|
|
23
|
+
* scoped to the pasted page's host or BGG's, never eBay's; no web search), so a seller is vetted from the source's own data (item
|
|
24
|
+
* 63), as signals with what the page showed, never as a verdict about a real person (plan section 6).
|
|
25
|
+
* A listing the model calls "no" is not DMed but is kept as a finding, so `/task history` shows
|
|
26
|
+
* it; one with no verdict is shown as "maybe". When judging fails for good, the listings go out
|
|
27
|
+
* unchecked: an alert never waits on the model for longer than its retries.
|
|
28
|
+
*
|
|
29
|
+
* The Job may not search the web or read the runner's files (`Read`, `Glob`, `Grep`, `LS`).
|
|
30
|
+
* Failures retry as the scout's do (`MAX_TRIES`, `auth_failed` an hour later); a usage limit never
|
|
31
|
+
* reaches `finish`. Model output is data: each field is capped, cleaned and made inert.
|
|
32
|
+
*/
|
|
33
|
+
export declare const FITS: readonly ["match", "maybe", "no"];
|
|
34
|
+
export type Fit = (typeof FITS)[number];
|
|
35
|
+
/** Proposed, like the scout's (item 61): a short look at up to 20 pages. */
|
|
36
|
+
export declare const JUDGE_MAX_TURNS = 12;
|
|
37
|
+
export declare const JUDGE_MAX_BUDGET_USD = 0.5;
|
|
38
|
+
export declare const JUDGE_TIMEOUT_MS = 300000;
|
|
39
|
+
/** How long past its due time a waiting retry is left to its own follow-up before a scheduled run takes it. */
|
|
40
|
+
export declare const RETRY_GRACE_MS: number;
|
|
41
|
+
export interface JudgeState extends WantState {
|
|
42
|
+
/** Listings read and not told yet, waiting for the model's look; at most `MAX_NEW_PER_RUN`. */
|
|
43
|
+
pending: Listing[];
|
|
44
|
+
/** Failed tries of the current look. */
|
|
45
|
+
failures: number;
|
|
46
|
+
/** When the waiting retry is due (ISO), while `failures` > 0. */
|
|
47
|
+
retryAt?: string;
|
|
48
|
+
}
|
|
49
|
+
export interface JudgeVerdict {
|
|
50
|
+
fit: Fit;
|
|
51
|
+
why: string;
|
|
52
|
+
seller: string;
|
|
53
|
+
}
|
|
54
|
+
export declare const JUDGE_SCHEMA: {
|
|
55
|
+
readonly type: "object";
|
|
56
|
+
readonly properties: {
|
|
57
|
+
readonly verdicts: {
|
|
58
|
+
readonly type: "array";
|
|
59
|
+
readonly items: {
|
|
60
|
+
readonly type: "object";
|
|
61
|
+
readonly properties: {
|
|
62
|
+
readonly n: {
|
|
63
|
+
readonly type: "integer";
|
|
64
|
+
readonly description: "The listing's number, as given.";
|
|
65
|
+
};
|
|
66
|
+
readonly fit: {
|
|
67
|
+
readonly type: "string";
|
|
68
|
+
readonly enum: readonly ["match", "maybe", "no"];
|
|
69
|
+
};
|
|
70
|
+
readonly why: {
|
|
71
|
+
readonly type: "string";
|
|
72
|
+
};
|
|
73
|
+
readonly seller: {
|
|
74
|
+
readonly type: "string";
|
|
75
|
+
};
|
|
76
|
+
};
|
|
77
|
+
readonly required: readonly ["n", "fit", "why"];
|
|
78
|
+
};
|
|
79
|
+
readonly maxItems: 20;
|
|
80
|
+
};
|
|
81
|
+
};
|
|
82
|
+
readonly required: readonly ["verdicts"];
|
|
83
|
+
};
|
|
84
|
+
export declare function judgeState(value: unknown): JudgeState;
|
|
85
|
+
/**
|
|
86
|
+
* The hosts the judge may open: only the watch's own -- the pasted page's host (or it with or
|
|
87
|
+
* without `www.`), or BGG's -- and only where a pending listing is on it. That host is one the
|
|
88
|
+
* tracker's own fenced reads reach every poll (the host's Fetch port checks its address after the name lookup);
|
|
89
|
+
* a host a listing merely names is never granted, since the runner sits on roshne's network and a
|
|
90
|
+
* shop's data could name a private one (`nas.lan`). A listing elsewhere is judged from its own
|
|
91
|
+
* text. Never an address, never eBay.
|
|
92
|
+
*/
|
|
93
|
+
export declare function judgeHosts(pending: readonly Listing[], config: WantConfig): string[];
|
|
94
|
+
/** The judge Job: the want, the listings numbered (shop text, inert, marked as data), and the rules. */
|
|
95
|
+
export declare function judgeJob(title: string, config: WantConfig, pending: readonly Listing[]): JobSpec;
|
|
96
|
+
/** The verdicts by listing number (1-based, up to `count`), the first for each; null when it is no answer. */
|
|
97
|
+
export declare function parseVerdicts(value: unknown, count: number): Map<number, JudgeVerdict> | null;
|
|
98
|
+
/** The DM: the listings worth a look, best first, each with the model's note and the seller signals. */
|
|
99
|
+
export declare function renderJudged(title: string, taskId: string, shown: readonly {
|
|
100
|
+
l: Listing;
|
|
101
|
+
v: JudgeVerdict;
|
|
102
|
+
}[], passed: number, config: WantConfig): string;
|
|
103
|
+
/** The DM when judging failed for good: the listings unchecked, so the alert still goes out. */
|
|
104
|
+
export declare function renderUnchecked(title: string, taskId: string, pending: readonly Listing[], reason: string, config: WantConfig): string;
|
|
105
|
+
export interface JudgeOptions {
|
|
106
|
+
/** How a deferred BGG read waits before its one second try (tests pass a no-op). */
|
|
107
|
+
wait?: (ms: number) => Promise<void>;
|
|
108
|
+
}
|
|
109
|
+
/** The judged type over `sources` (the plain watch's); BGG's spacing is waited out once, on this background lane. */
|
|
110
|
+
export declare function wantjudgeType(sources: Partial<Record<SourceId, Source>>, o?: JudgeOptions): TaskType<WantConfig>;
|
|
111
|
+
//# sourceMappingURL=wantjudge.d.ts.map
|