@rackbops/plugin-tracker 0.35.0 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/plugin.js +607 -433
- package/package.json +3 -3
package/dist/plugin.js
CHANGED
|
@@ -192,15 +192,15 @@ function reads(instant, zone, d, h, min) {
|
|
|
192
192
|
}
|
|
193
193
|
function zonedInstant(year, month, day, hour, minute, zone) {
|
|
194
194
|
const wanted = Date.UTC(year, month - 1, day, hour, minute);
|
|
195
|
-
const
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
const
|
|
200
|
-
const
|
|
201
|
-
if (
|
|
202
|
-
return new Date(
|
|
203
|
-
return new Date(Math.max(
|
|
195
|
+
const offsets = new Set;
|
|
196
|
+
for (const at of [wanted - 86400000, wanted, wanted + 86400000]) {
|
|
197
|
+
offsets.add(zoneOffsetMinutes(new Date(at), zone));
|
|
198
|
+
}
|
|
199
|
+
const candidates = [...offsets].map((offset) => wanted - offset * 60000);
|
|
200
|
+
const matching = candidates.filter((c) => reads(c, zone, day, hour, minute));
|
|
201
|
+
if (matching.length > 0)
|
|
202
|
+
return new Date(Math.min(...matching));
|
|
203
|
+
return new Date(Math.max(...candidates));
|
|
204
204
|
}
|
|
205
205
|
|
|
206
206
|
// node_modules/@rackbops/docket-core/dist/budget.js
|
|
@@ -223,16 +223,20 @@ function budgetDay(now) {
|
|
|
223
223
|
end: end.toISOString()
|
|
224
224
|
};
|
|
225
225
|
}
|
|
226
|
+
function costOf(usd) {
|
|
227
|
+
return typeof usd === "number" && Number.isFinite(usd) && usd > 0 ? usd : 0;
|
|
228
|
+
}
|
|
226
229
|
function spent(rows) {
|
|
227
|
-
return rows.reduce((s, r) => ({ usd: s.usd + r.costUsd, calls: s.calls + r.calls }), {
|
|
230
|
+
return rows.reduce((s, r) => ({ usd: s.usd + costOf(r.costUsd), calls: s.calls + r.calls }), {
|
|
228
231
|
usd: 0,
|
|
229
232
|
calls: 0
|
|
230
233
|
});
|
|
231
234
|
}
|
|
235
|
+
var USD_EPSILON = 0.000000001;
|
|
232
236
|
function reached(used, limits) {
|
|
233
237
|
if (limits.calls !== null && used.calls >= limits.calls)
|
|
234
238
|
return "calls";
|
|
235
|
-
if (limits.usd !== null && used.usd >= limits.usd)
|
|
239
|
+
if (limits.usd !== null && used.usd >= limits.usd - USD_EPSILON)
|
|
236
240
|
return "usd";
|
|
237
241
|
return null;
|
|
238
242
|
}
|
|
@@ -261,7 +265,7 @@ async function charge(store, c) {
|
|
|
261
265
|
source: c.source,
|
|
262
266
|
key: c.key ?? null,
|
|
263
267
|
calls: c.calls,
|
|
264
|
-
costUsd: c.costUsd
|
|
268
|
+
costUsd: costOf(c.costUsd),
|
|
265
269
|
at: c.at.toISOString()
|
|
266
270
|
});
|
|
267
271
|
}
|
|
@@ -426,6 +430,10 @@ async function respondToInvite(store, task, recipientId, answer, now, occurrence
|
|
|
426
430
|
return { recipient, reply, block };
|
|
427
431
|
}
|
|
428
432
|
async function optOut(store, task, recipientId, now, occurrenceId = null) {
|
|
433
|
+
const existing = (await store.listRecipients(task.id)).find((r) => r.userId === recipientId);
|
|
434
|
+
if (existing?.state !== "accepted") {
|
|
435
|
+
throw new ConsentError(`${recipientId} is not an accepted recipient of task ${task.id}`);
|
|
436
|
+
}
|
|
429
437
|
const at = now.toISOString();
|
|
430
438
|
await store.setRecipient(task.id, recipientId, "opted_out", at);
|
|
431
439
|
await store.addTaskEvent({
|
|
@@ -459,6 +467,22 @@ function isLane(value) {
|
|
|
459
467
|
return LANES.includes(value);
|
|
460
468
|
}
|
|
461
469
|
|
|
470
|
+
// node_modules/@rackbops/docket-core/dist/local-date.js
|
|
471
|
+
var DAY_MS = 86400000;
|
|
472
|
+
function daysInMonth(year, month) {
|
|
473
|
+
return new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
474
|
+
}
|
|
475
|
+
function dayNumber(d) {
|
|
476
|
+
return Math.floor(Date.UTC(d.year, d.month - 1, d.day) / DAY_MS);
|
|
477
|
+
}
|
|
478
|
+
function fromDayNumber(n) {
|
|
479
|
+
const dt = new Date(n * DAY_MS);
|
|
480
|
+
return { year: dt.getUTCFullYear(), month: dt.getUTCMonth() + 1, day: dt.getUTCDate() };
|
|
481
|
+
}
|
|
482
|
+
function addDays(d, days) {
|
|
483
|
+
return fromDayNumber(dayNumber(d) + days);
|
|
484
|
+
}
|
|
485
|
+
|
|
462
486
|
// node_modules/@rackbops/docket-core/dist/schedule.js
|
|
463
487
|
var SCHEDULE_KINDS = ["once", "calendar", "poll", "period"];
|
|
464
488
|
function isScheduleKind(value) {
|
|
@@ -466,8 +490,12 @@ function isScheduleKind(value) {
|
|
|
466
490
|
}
|
|
467
491
|
var MIN_POLL_MINUTES = 5;
|
|
468
492
|
var MINUTE_MS = 60000;
|
|
469
|
-
var DAY_MS = 86400000;
|
|
470
493
|
var HORIZON_DAYS = 366 * 3;
|
|
494
|
+
var MAX_CALENDAR_EVERY = {
|
|
495
|
+
day: HORIZON_DAYS - 1,
|
|
496
|
+
week: Math.floor((HORIZON_DAYS - 1) / 7),
|
|
497
|
+
month: 36
|
|
498
|
+
};
|
|
471
499
|
var HORIZON_PERIODS = 1e5;
|
|
472
500
|
function parseLocalDate(value) {
|
|
473
501
|
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
|
|
@@ -485,16 +513,6 @@ function formatLocalDate(d) {
|
|
|
485
513
|
const dd = String(d.day).padStart(2, "0");
|
|
486
514
|
return `${d.year}-${mm}-${dd}`;
|
|
487
515
|
}
|
|
488
|
-
function daysInMonth(year, month) {
|
|
489
|
-
return new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
490
|
-
}
|
|
491
|
-
function dayNumber(d) {
|
|
492
|
-
return Math.floor(Date.UTC(d.year, d.month - 1, d.day) / DAY_MS);
|
|
493
|
-
}
|
|
494
|
-
function fromDayNumber(n) {
|
|
495
|
-
const dt = new Date(n * DAY_MS);
|
|
496
|
-
return { year: dt.getUTCFullYear(), month: dt.getUTCMonth() + 1, day: dt.getUTCDate() };
|
|
497
|
-
}
|
|
498
516
|
function weekdayOf(d) {
|
|
499
517
|
return new Date(Date.UTC(d.year, d.month - 1, d.day)).getUTCDay();
|
|
500
518
|
}
|
|
@@ -536,8 +554,14 @@ function calendarProblems(schedule) {
|
|
|
536
554
|
if (!Number.isInteger(schedule.every) || schedule.every < 1) {
|
|
537
555
|
problems.push("calendar.every must be a whole number >= 1");
|
|
538
556
|
}
|
|
539
|
-
if (!["day", "week", "month"].includes(schedule.unit))
|
|
557
|
+
if (!["day", "week", "month"].includes(schedule.unit)) {
|
|
540
558
|
problems.push("calendar.unit is unknown");
|
|
559
|
+
} else if (Number.isInteger(schedule.every) && schedule.every >= 1) {
|
|
560
|
+
const max = MAX_CALENDAR_EVERY[schedule.unit];
|
|
561
|
+
if (schedule.every > max) {
|
|
562
|
+
problems.push(`calendar.every must be at most ${max} for unit ${schedule.unit}`);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
541
565
|
if (!parseLocalDate(schedule.start))
|
|
542
566
|
problems.push("calendar.start must be a YYYY-MM-DD date");
|
|
543
567
|
if (schedule.weekdays !== undefined) {
|
|
@@ -659,11 +683,23 @@ function nextPollDue(schedule, after) {
|
|
|
659
683
|
return null;
|
|
660
684
|
return new Date(due);
|
|
661
685
|
}
|
|
686
|
+
var LONGEST_PERIOD_DAYS = { day: 1, week: 7, month: 31, year: 366 };
|
|
687
|
+
function firstPeriod(schedule, after, zone) {
|
|
688
|
+
const anchor = parseLocalDate(schedule.anchor);
|
|
689
|
+
if (!anchor)
|
|
690
|
+
return 0;
|
|
691
|
+
const local = wallClock(after, zone);
|
|
692
|
+
const elapsed = dayNumber({ year: local.year, month: local.month, day: local.day }) - dayNumber(anchor);
|
|
693
|
+
const longest = LONGEST_PERIOD_DAYS[schedule.unit] * schedule.every;
|
|
694
|
+
if (!(longest >= 1) || (schedule.leadDays ?? 0) < 0)
|
|
695
|
+
return 0;
|
|
696
|
+
return Math.max(0, Math.floor(elapsed / longest) - 1);
|
|
697
|
+
}
|
|
662
698
|
function nextPeriodDue(schedule, after, ctx) {
|
|
663
699
|
const hour = schedule.hour ?? ctx.preferredHour;
|
|
664
700
|
const minute = schedule.minute ?? 0;
|
|
665
701
|
const lead = schedule.leadDays ?? 0;
|
|
666
|
-
for (let n =
|
|
702
|
+
for (let n = firstPeriod(schedule, after, ctx.zone);n < HORIZON_PERIODS; n++) {
|
|
667
703
|
const date = periodDateAt(schedule, n);
|
|
668
704
|
if (!date)
|
|
669
705
|
return null;
|
|
@@ -805,8 +841,8 @@ async function deliver(store, notifier, occurrence, ownerId, targets, message, o
|
|
|
805
841
|
}
|
|
806
842
|
const target = targets.get(row.userId);
|
|
807
843
|
if (!target) {
|
|
808
|
-
await finalFail(store, row, "no longer a recipient", now());
|
|
809
|
-
report.failed.push(row.userId);
|
|
844
|
+
const ended = await finalFail(store, row, "no longer a recipient", now());
|
|
845
|
+
(ended ? report.failed : report.untried).push(row.userId);
|
|
810
846
|
continue;
|
|
811
847
|
}
|
|
812
848
|
const claim = await store.claimDelivery(occurrence.id, row.userId, now().toISOString());
|
|
@@ -861,18 +897,20 @@ async function sendOne(store, notifier, occurrence, claim, target, ownerId, mess
|
|
|
861
897
|
report.sent.push(claim.userId);
|
|
862
898
|
}
|
|
863
899
|
async function finalFail(store, row, error, at) {
|
|
864
|
-
await store.settleDelivery(row.occurrenceId, row.userId, {
|
|
900
|
+
const after = await store.settleDelivery(row.occurrenceId, row.userId, {
|
|
865
901
|
status: "failed",
|
|
866
902
|
error,
|
|
867
903
|
attempts: row.attempts,
|
|
868
904
|
deferrals: row.deferrals,
|
|
869
905
|
retryAt: null,
|
|
870
906
|
at: at.toISOString()
|
|
871
|
-
});
|
|
907
|
+
}, row.status);
|
|
908
|
+
return after.status === "failed" && after.settledAt === at.toISOString();
|
|
872
909
|
}
|
|
873
910
|
async function dropOwed(store, rows, error, at) {
|
|
874
911
|
for (const row of rows)
|
|
875
|
-
|
|
912
|
+
if (row.retryAt !== null)
|
|
913
|
+
await finalFail(store, row, error, at);
|
|
876
914
|
}
|
|
877
915
|
async function settleStaleClaims(store, now, all = false) {
|
|
878
916
|
const cutoff = new Date(now.getTime() - STALE_CLAIM_MS).toISOString();
|
|
@@ -880,15 +918,16 @@ async function settleStaleClaims(store, now, all = false) {
|
|
|
880
918
|
for (const row of await store.listDeliveries({ status: "claimed" })) {
|
|
881
919
|
if (!all && (row.claimedAt ?? "") > cutoff)
|
|
882
920
|
continue;
|
|
883
|
-
await store.settleDelivery(row.occurrenceId, row.userId, {
|
|
921
|
+
const after = await store.settleDelivery(row.occurrenceId, row.userId, {
|
|
884
922
|
status: "unconfirmed",
|
|
885
923
|
error: "claimed, never settled",
|
|
886
924
|
attempts: row.attempts,
|
|
887
925
|
deferrals: row.deferrals,
|
|
888
926
|
retryAt: null,
|
|
889
927
|
at: now.toISOString()
|
|
890
|
-
});
|
|
891
|
-
|
|
928
|
+
}, "claimed");
|
|
929
|
+
if (after.status === "unconfirmed" && after.settledAt === now.toISOString())
|
|
930
|
+
settled += 1;
|
|
892
931
|
}
|
|
893
932
|
return settled;
|
|
894
933
|
}
|
|
@@ -904,7 +943,7 @@ async function freshRoundForPause(store, rows) {
|
|
|
904
943
|
deferrals: 0,
|
|
905
944
|
retryAt: row.retryAt,
|
|
906
945
|
at: row.settledAt ?? row.createdAt
|
|
907
|
-
});
|
|
946
|
+
}, row.status);
|
|
908
947
|
}
|
|
909
948
|
}
|
|
910
949
|
|
|
@@ -1002,6 +1041,14 @@ var LATE_AFTER_MS = 5 * 60000;
|
|
|
1002
1041
|
class ScheduleError extends Error {
|
|
1003
1042
|
name = "ScheduleError";
|
|
1004
1043
|
}
|
|
1044
|
+
function checkSchedule(schedule, type) {
|
|
1045
|
+
if (type && !type.schedule.includes(schedule.kind)) {
|
|
1046
|
+
throw new ScheduleError(`${type.id} does not accept a ${schedule.kind} schedule`);
|
|
1047
|
+
}
|
|
1048
|
+
const problems = scheduleProblems(schedule);
|
|
1049
|
+
if (problems.length > 0)
|
|
1050
|
+
throw new ScheduleError(problems.join("; "));
|
|
1051
|
+
}
|
|
1005
1052
|
function isLate(occurrence, now) {
|
|
1006
1053
|
return now.getTime() - Date.parse(occurrence.dueAt) > LATE_AFTER_MS;
|
|
1007
1054
|
}
|
|
@@ -1037,13 +1084,19 @@ function firstDueAfter(schedule, fresh, now) {
|
|
|
1037
1084
|
async function cancelScheduledRuns(store, taskId) {
|
|
1038
1085
|
let removed = 0;
|
|
1039
1086
|
for (const o of await store.listOccurrences({ taskId, status: "queued" })) {
|
|
1040
|
-
if (isOffSchedule(o.dedupeKey)
|
|
1087
|
+
if (isOffSchedule(o.dedupeKey))
|
|
1041
1088
|
continue;
|
|
1042
|
-
if (await store
|
|
1089
|
+
if (await deleteIdleRun(store, o))
|
|
1043
1090
|
removed += 1;
|
|
1044
1091
|
}
|
|
1045
1092
|
return removed;
|
|
1046
1093
|
}
|
|
1094
|
+
async function deleteIdleRun(store, o) {
|
|
1095
|
+
const lastEventId = (await store.listEvents(o.id)).at(-1)?.id ?? null;
|
|
1096
|
+
if (hasFired(o) || await hasJobOut(store, o))
|
|
1097
|
+
return false;
|
|
1098
|
+
return store.deleteOccurrence(o.id, { lastEventId });
|
|
1099
|
+
}
|
|
1047
1100
|
async function anyJobOut(store, taskId) {
|
|
1048
1101
|
for (const o of await store.listOccurrences({ taskId, status: "queued" })) {
|
|
1049
1102
|
if (!hasFired(o) && await hasJobOut(store, o))
|
|
@@ -1051,12 +1104,9 @@ async function anyJobOut(store, taskId) {
|
|
|
1051
1104
|
}
|
|
1052
1105
|
return false;
|
|
1053
1106
|
}
|
|
1054
|
-
async function reschedule(store, task, owner, schedule, actorId, now) {
|
|
1055
|
-
if (schedule)
|
|
1056
|
-
|
|
1057
|
-
if (problems.length > 0)
|
|
1058
|
-
throw new ScheduleError(problems.join("; "));
|
|
1059
|
-
}
|
|
1107
|
+
async function reschedule(store, task, owner, schedule, actorId, now, type) {
|
|
1108
|
+
if (schedule)
|
|
1109
|
+
checkSchedule(schedule, type);
|
|
1060
1110
|
const jobOut = await anyJobOut(store, task.id);
|
|
1061
1111
|
if (jobOut && (schedule?.kind === "once" || task.schedule?.kind === "once")) {
|
|
1062
1112
|
throw new ScheduleError("its run is with the runner now; change when it runs once that run is back");
|
|
@@ -1094,7 +1144,11 @@ function clean(text, max, oneLine = false) {
|
|
|
1094
1144
|
if (oneLine)
|
|
1095
1145
|
out = out.replace(/\s+/g, " ");
|
|
1096
1146
|
out = out.trim();
|
|
1097
|
-
|
|
1147
|
+
if (out.length <= max)
|
|
1148
|
+
return out;
|
|
1149
|
+
if (max < 4)
|
|
1150
|
+
return out.slice(0, Math.max(0, max));
|
|
1151
|
+
return `${out.slice(0, max - 3).trimEnd()}...`;
|
|
1098
1152
|
}
|
|
1099
1153
|
|
|
1100
1154
|
// node_modules/@rackbops/docket-core/dist/dispatch.js
|
|
@@ -1223,10 +1277,14 @@ class Lanes {
|
|
|
1223
1277
|
if ("error" in loaded || !loaded.type.onReply)
|
|
1224
1278
|
return null;
|
|
1225
1279
|
const occurrence = input.occurrenceId ? await store.getOccurrence(input.occurrenceId) : null;
|
|
1226
|
-
if (!occurrence)
|
|
1280
|
+
if (!occurrence || occurrence.taskId !== task.id)
|
|
1227
1281
|
return null;
|
|
1228
1282
|
const ctx = await this.context(loaded, occurrence, now);
|
|
1229
|
-
const outcome = await loaded.type.onReply({ ...ctx, reply });
|
|
1283
|
+
const outcome = { ...await loaded.type.onReply({ ...ctx, reply }) };
|
|
1284
|
+
if (input.userId !== task.ownerId) {
|
|
1285
|
+
delete outcome.complete;
|
|
1286
|
+
delete outcome.snoozeUntil;
|
|
1287
|
+
}
|
|
1230
1288
|
await this.apply(task, outcome, now, null, `reply:${reply.id}`);
|
|
1231
1289
|
if (outcome.complete)
|
|
1232
1290
|
await this.complete(task, input.userId, now);
|
|
@@ -1375,8 +1433,7 @@ class Lanes {
|
|
|
1375
1433
|
const { store } = this.d;
|
|
1376
1434
|
const at = now.toISOString();
|
|
1377
1435
|
for (const o of await store.listOccurrences({ taskId: task.id, status: "queued" })) {
|
|
1378
|
-
|
|
1379
|
-
await store.deleteOccurrence(o.id);
|
|
1436
|
+
await deleteIdleRun(store, o);
|
|
1380
1437
|
}
|
|
1381
1438
|
await store.updateTask(task.id, { status: "done", at });
|
|
1382
1439
|
await store.addTaskEvent({ taskId: task.id, actorId, kind: "completed", detail: "", at });
|
|
@@ -1996,14 +2053,8 @@ async function createTask(store, actor, owner, input, now) {
|
|
|
1996
2053
|
if (!actor.admin && actor.userId !== owner.id) {
|
|
1997
2054
|
throw new TaskError("only the owner or an admin creates a task for a person");
|
|
1998
2055
|
}
|
|
1999
|
-
if (input.schedule)
|
|
2000
|
-
|
|
2001
|
-
throw new ScheduleError(`${input.type.id} does not accept a ${input.schedule.kind} schedule`);
|
|
2002
|
-
}
|
|
2003
|
-
const problems = scheduleProblems(input.schedule);
|
|
2004
|
-
if (problems.length > 0)
|
|
2005
|
-
throw new ScheduleError(problems.join("; "));
|
|
2006
|
-
}
|
|
2056
|
+
if (input.schedule)
|
|
2057
|
+
checkSchedule(input.schedule, input.type);
|
|
2007
2058
|
const at = now.toISOString();
|
|
2008
2059
|
const task = await store.createTask({
|
|
2009
2060
|
ownerId: owner.id,
|
|
@@ -2053,27 +2104,12 @@ var DAY_TIME_RE = new RegExp(`^(?:on )?(${DAY})(?: (?:at )?(${TIME}))?$`);
|
|
|
2053
2104
|
var TIME_DAY_RE = new RegExp(`^(?:at )?(${TIME})(?: (?:on )?(${DAY}))?$`);
|
|
2054
2105
|
var MAX_AHEAD_YEARS = 5;
|
|
2055
2106
|
var MAX_AHEAD_MS = MAX_AHEAD_YEARS * 365.25 * 86400000;
|
|
2056
|
-
function knownZone(zone) {
|
|
2057
|
-
try {
|
|
2058
|
-
new Intl.DateTimeFormat("en-US", { timeZone: zone });
|
|
2059
|
-
return true;
|
|
2060
|
-
} catch {
|
|
2061
|
-
return false;
|
|
2062
|
-
}
|
|
2063
|
-
}
|
|
2064
2107
|
var HINT = `Try "in 2 hours", "9am", "tomorrow 9am", "friday 17:30" or "2026-10-01 9:00".`;
|
|
2065
2108
|
function fail(text, why) {
|
|
2066
2109
|
return { ok: false, error: `${why ?? `I couldn't read "${text}" as a time.`} ${HINT}` };
|
|
2067
2110
|
}
|
|
2068
|
-
function daysIn(year, month) {
|
|
2069
|
-
return new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
2070
|
-
}
|
|
2071
|
-
function addDays(date, days) {
|
|
2072
|
-
const d = new Date(Date.UTC(date.year, date.month - 1, date.day + days));
|
|
2073
|
-
return { year: d.getUTCFullYear(), month: d.getUTCMonth() + 1, day: d.getUTCDate() };
|
|
2074
|
-
}
|
|
2075
2111
|
function validDate(date) {
|
|
2076
|
-
return date.month >= 1 && date.month <= 12 && date.day >= 1 && date.day <=
|
|
2112
|
+
return date.month >= 1 && date.month <= 12 && date.day >= 1 && date.day <= daysInMonth(date.year, date.month);
|
|
2077
2113
|
}
|
|
2078
2114
|
function parseTime(word) {
|
|
2079
2115
|
if (word === "noon")
|
|
@@ -2132,7 +2168,7 @@ function resolve(text, now, options) {
|
|
|
2132
2168
|
return fail(text, `${text} is more than ${MAX_AHEAD_YEARS} years away.`);
|
|
2133
2169
|
return { ok: true, at: new Date(now.getTime() + n * unit) };
|
|
2134
2170
|
}
|
|
2135
|
-
if (!
|
|
2171
|
+
if (!isTimeZone(options.zone))
|
|
2136
2172
|
return fail(text, `I don't know the time zone ${options.zone}.`);
|
|
2137
2173
|
let dayWord;
|
|
2138
2174
|
let timeWord;
|
|
@@ -2283,7 +2319,7 @@ function priceFromJson(value, depth = 0) {
|
|
|
2283
2319
|
}
|
|
2284
2320
|
return null;
|
|
2285
2321
|
}
|
|
2286
|
-
var JSON_LD = /<script[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
|
|
2322
|
+
var JSON_LD = /<script[^>]*type\s*=\s*(?:["']application\/ld\+json["']|application\/ld\+json(?=[\s>]))[^>]*>([\s\S]*?)<\/script>/gi;
|
|
2287
2323
|
function jsonLdBlocks(html) {
|
|
2288
2324
|
const blocks = [];
|
|
2289
2325
|
for (const m of html.matchAll(JSON_LD)) {
|
|
@@ -2318,14 +2354,85 @@ function priceFromMeta(html) {
|
|
|
2318
2354
|
}
|
|
2319
2355
|
return null;
|
|
2320
2356
|
}
|
|
2321
|
-
|
|
2322
|
-
|
|
2357
|
+
var MAX_PATTERN_CHARS = 300;
|
|
2358
|
+
var MAX_PATTERN_BODY_CHARS = 500000;
|
|
2359
|
+
function patternProblem(pattern) {
|
|
2360
|
+
if (pattern.length > MAX_PATTERN_CHARS) {
|
|
2361
|
+
return `the pattern is longer than ${MAX_PATTERN_CHARS} characters`;
|
|
2362
|
+
}
|
|
2323
2363
|
try {
|
|
2324
|
-
|
|
2364
|
+
new RegExp(pattern, "i");
|
|
2325
2365
|
} catch {
|
|
2326
|
-
return
|
|
2366
|
+
return "the pattern is not a valid regular expression";
|
|
2367
|
+
}
|
|
2368
|
+
return repeatsAnAmbiguousGroup(pattern) ? "the pattern repeats a group that itself repeats or has a |, like (a+)+ or (a|b)*, " + "which can stall the tracker; use a character class, like [\\d,]+" : null;
|
|
2369
|
+
}
|
|
2370
|
+
function unboundedAt(p, i) {
|
|
2371
|
+
const c = p[i];
|
|
2372
|
+
return c === "*" || c === "+" || /^\{\d+,\}/.test(p.slice(i));
|
|
2373
|
+
}
|
|
2374
|
+
function repeatsAt(p, i) {
|
|
2375
|
+
if (unboundedAt(p, i))
|
|
2376
|
+
return true;
|
|
2377
|
+
const m = /^\{(\d+)(?:,(\d+))?\}/.exec(p.slice(i));
|
|
2378
|
+
return m !== null && Number(m[2] ?? m[1]) > 1;
|
|
2379
|
+
}
|
|
2380
|
+
function repeatsAnAmbiguousGroup(p) {
|
|
2381
|
+
const open = [];
|
|
2382
|
+
const top = () => open[open.length - 1];
|
|
2383
|
+
let inside = false;
|
|
2384
|
+
for (let i = 0;i < p.length; i++) {
|
|
2385
|
+
const c = p[i];
|
|
2386
|
+
if (c === "\\") {
|
|
2387
|
+
i++;
|
|
2388
|
+
continue;
|
|
2389
|
+
}
|
|
2390
|
+
if (inside) {
|
|
2391
|
+
if (c === "]")
|
|
2392
|
+
inside = false;
|
|
2393
|
+
continue;
|
|
2394
|
+
}
|
|
2395
|
+
if (c === "[") {
|
|
2396
|
+
inside = true;
|
|
2397
|
+
if (p[i + 1] === "]")
|
|
2398
|
+
i++;
|
|
2399
|
+
continue;
|
|
2400
|
+
}
|
|
2401
|
+
if (c === "(") {
|
|
2402
|
+
open.push({ repeats: false, alt: false });
|
|
2403
|
+
continue;
|
|
2404
|
+
}
|
|
2405
|
+
if (c === "|") {
|
|
2406
|
+
const g = top();
|
|
2407
|
+
if (g)
|
|
2408
|
+
g.alt = true;
|
|
2409
|
+
continue;
|
|
2410
|
+
}
|
|
2411
|
+
if (c === ")") {
|
|
2412
|
+
const g = open.pop() ?? { repeats: false, alt: false };
|
|
2413
|
+
const repeated = repeatsAt(p, i + 1);
|
|
2414
|
+
if (repeated && (g.repeats || g.alt))
|
|
2415
|
+
return true;
|
|
2416
|
+
const parent = top();
|
|
2417
|
+
if (parent) {
|
|
2418
|
+
if (g.repeats || unboundedAt(p, i + 1))
|
|
2419
|
+
parent.repeats = true;
|
|
2420
|
+
if (g.alt && !repeated)
|
|
2421
|
+
parent.alt = true;
|
|
2422
|
+
}
|
|
2423
|
+
continue;
|
|
2424
|
+
}
|
|
2425
|
+
const g = top();
|
|
2426
|
+
if (g && unboundedAt(p, i))
|
|
2427
|
+
g.repeats = true;
|
|
2327
2428
|
}
|
|
2328
|
-
|
|
2429
|
+
return false;
|
|
2430
|
+
}
|
|
2431
|
+
function priceFromPattern(body, pattern) {
|
|
2432
|
+
if (patternProblem(pattern) !== null)
|
|
2433
|
+
return null;
|
|
2434
|
+
const re = new RegExp(pattern, "i");
|
|
2435
|
+
const m = re.exec(body.slice(0, MAX_PATTERN_BODY_CHARS));
|
|
2329
2436
|
const captured = m?.[1] ?? m?.[0];
|
|
2330
2437
|
if (captured === undefined)
|
|
2331
2438
|
return null;
|
|
@@ -2355,8 +2462,19 @@ function extractPrice(body, pattern) {
|
|
|
2355
2462
|
}
|
|
2356
2463
|
|
|
2357
2464
|
// node_modules/@rackbops/docket-types/dist/money.js
|
|
2465
|
+
var ZERO_DECIMAL = new Set("BIF CLP DJF GNF ISK JPY KMF KRW PYG RWF UGX UYI VND VUV XAF XOF XPF".split(" "));
|
|
2466
|
+
var THREE_DECIMAL = new Set("BHD IQD JOD KWD LYD OMR TND".split(" "));
|
|
2467
|
+
function currencyCode(raw) {
|
|
2468
|
+
const c = (raw ?? "").trim().toUpperCase();
|
|
2469
|
+
return /^[A-Z]{3}$/.test(c) ? c : undefined;
|
|
2470
|
+
}
|
|
2471
|
+
function minorUnits(currency) {
|
|
2472
|
+
const c = currency.trim().toUpperCase();
|
|
2473
|
+
return ZERO_DECIMAL.has(c) ? 1 : THREE_DECIMAL.has(c) ? 1000 : 100;
|
|
2474
|
+
}
|
|
2358
2475
|
function money(amount, currency) {
|
|
2359
|
-
|
|
2476
|
+
const digits = Math.round(Math.log10(minorUnits(currency)));
|
|
2477
|
+
return `${amount.toFixed(digits)} ${currency}`.trim();
|
|
2360
2478
|
}
|
|
2361
2479
|
|
|
2362
2480
|
// node_modules/@rackbops/docket-types/dist/price.js
|
|
@@ -2382,7 +2500,8 @@ function stateOf(value) {
|
|
|
2382
2500
|
peak: s.peak ?? null,
|
|
2383
2501
|
below: s.below ?? false,
|
|
2384
2502
|
misses: s.misses,
|
|
2385
|
-
alerts: s.alerts ?? 0
|
|
2503
|
+
alerts: s.alerts ?? 0,
|
|
2504
|
+
...typeof s.currency === "string" && s.currency !== "" ? { currency: s.currency } : {}
|
|
2386
2505
|
};
|
|
2387
2506
|
}
|
|
2388
2507
|
function dropPercent(config) {
|
|
@@ -2441,6 +2560,9 @@ var price = defineTaskType({
|
|
|
2441
2560
|
if (!fetch2)
|
|
2442
2561
|
throw new Error("the price type needs the Fetch port; the host configured none");
|
|
2443
2562
|
const state = stateOf(ctx.state);
|
|
2563
|
+
const refused = ctx.config.pattern ? patternProblem(ctx.config.pattern) : null;
|
|
2564
|
+
if (refused)
|
|
2565
|
+
return miss(ctx, state, `pattern refused: ${refused}`);
|
|
2444
2566
|
let body;
|
|
2445
2567
|
try {
|
|
2446
2568
|
const response = await fetch2.get(ctx.config.url);
|
|
@@ -2458,12 +2580,14 @@ var price = defineTaskType({
|
|
|
2458
2580
|
const seen = found.value;
|
|
2459
2581
|
const currency = found.currency ?? ctx.config.currency ?? "";
|
|
2460
2582
|
const point = currency ? { value: seen, unit: currency, note: "observed" } : { value: seen, note: "observed" };
|
|
2583
|
+
const code = currencyCode(currency) ?? "";
|
|
2584
|
+
const kept = code === "" ? {} : { currency: code };
|
|
2461
2585
|
const rule = baselineRule(ctx.config);
|
|
2462
2586
|
const drop = dropPercent(ctx.config);
|
|
2463
2587
|
const ref = reference(state, rule);
|
|
2464
2588
|
if (ref === null) {
|
|
2465
2589
|
return {
|
|
2466
|
-
state: { first: seen, last: seen, peak: seen, below: false, misses: 0, alerts: 0 },
|
|
2590
|
+
state: { first: seen, last: seen, peak: seen, below: false, misses: 0, alerts: 0, ...kept },
|
|
2467
2591
|
series: [point],
|
|
2468
2592
|
notify: {
|
|
2469
2593
|
text: `Now tracking ${ctx.task.title} at ${money(seen, currency)}. I will say when it ` + `drops ${drop}% or more from the ${rule} price seen. ${ctx.config.url}`,
|
|
@@ -2472,6 +2596,11 @@ var price = defineTaskType({
|
|
|
2472
2596
|
summary: `baseline ${money(seen, currency)}`
|
|
2473
2597
|
};
|
|
2474
2598
|
}
|
|
2599
|
+
const declared = currencyCode(found.currency) ?? "";
|
|
2600
|
+
const known = state.currency ?? currencyCode(ctx.config.currency) ?? "";
|
|
2601
|
+
if (known !== "" && declared !== "" && declared !== known) {
|
|
2602
|
+
return miss(ctx, state, `the page now prices in ${declared}, not ${known}`);
|
|
2603
|
+
}
|
|
2475
2604
|
const line = ref * (1 - drop / 100);
|
|
2476
2605
|
const below = seen <= line + 0.000000001;
|
|
2477
2606
|
const crossing = below && !state.below;
|
|
@@ -2481,7 +2610,8 @@ var price = defineTaskType({
|
|
|
2481
2610
|
peak: Math.max(state.peak ?? seen, seen),
|
|
2482
2611
|
below,
|
|
2483
2612
|
misses: 0,
|
|
2484
|
-
alerts: state.alerts + (crossing ? 1 : 0)
|
|
2613
|
+
alerts: state.alerts + (crossing ? 1 : 0),
|
|
2614
|
+
...state.currency ? { currency: state.currency } : kept
|
|
2485
2615
|
};
|
|
2486
2616
|
if (!crossing) {
|
|
2487
2617
|
return {
|
|
@@ -2513,8 +2643,9 @@ var DEFAULT_SNOOZE_MS = 60 * 60000;
|
|
|
2513
2643
|
function snoozeUntil(payload, now) {
|
|
2514
2644
|
if (typeof payload === "object" && payload !== null) {
|
|
2515
2645
|
const p = payload;
|
|
2516
|
-
|
|
2517
|
-
|
|
2646
|
+
const until = typeof p.until === "string" ? Date.parse(p.until) : Number.NaN;
|
|
2647
|
+
if (!Number.isNaN(until) && until > now.getTime())
|
|
2648
|
+
return new Date(until);
|
|
2518
2649
|
if (typeof p.minutes === "number" && Number.isFinite(p.minutes) && p.minutes > 0) {
|
|
2519
2650
|
return new Date(now.getTime() + p.minutes * 60000);
|
|
2520
2651
|
}
|
|
@@ -2761,7 +2892,10 @@ function safeUrl(value) {
|
|
|
2761
2892
|
if (url.username || url.password)
|
|
2762
2893
|
return null;
|
|
2763
2894
|
const text = url.toString();
|
|
2764
|
-
|
|
2895
|
+
const decoded = decodeURIComponentSafe(text);
|
|
2896
|
+
if (/@(everyone|here)\b|<[@#]/i.test(decoded))
|
|
2897
|
+
return null;
|
|
2898
|
+
if (decoded.includes("]("))
|
|
2765
2899
|
return null;
|
|
2766
2900
|
return text;
|
|
2767
2901
|
}
|
|
@@ -2783,7 +2917,7 @@ function parseAnswer(value) {
|
|
|
2783
2917
|
if (summary.length === 0)
|
|
2784
2918
|
return null;
|
|
2785
2919
|
const findings = [];
|
|
2786
|
-
const
|
|
2920
|
+
const unsourced = [];
|
|
2787
2921
|
for (const f of value.findings) {
|
|
2788
2922
|
if (!isObj2(f) || typeof f.claim !== "string")
|
|
2789
2923
|
continue;
|
|
@@ -2793,10 +2927,11 @@ function parseAnswer(value) {
|
|
|
2793
2927
|
const raw = Array.isArray(f.sources) ? f.sources : [];
|
|
2794
2928
|
const sources = [...new Set(raw.map(safeUrl).filter((u) => u !== null))];
|
|
2795
2929
|
if (sources.length === 0)
|
|
2796
|
-
|
|
2930
|
+
unsourced.push(`${claim} (no source given)`);
|
|
2797
2931
|
else
|
|
2798
2932
|
findings.push({ claim, sources: sources.slice(0, MAX_SOURCES_PER_FINDING) });
|
|
2799
2933
|
}
|
|
2934
|
+
const uncertain = [];
|
|
2800
2935
|
const said = Array.isArray(value.uncertain) ? value.uncertain : [];
|
|
2801
2936
|
for (const u of said) {
|
|
2802
2937
|
if (typeof u !== "string")
|
|
@@ -2805,6 +2940,7 @@ function parseAnswer(value) {
|
|
|
2805
2940
|
if (line.length > 0)
|
|
2806
2941
|
uncertain.push(line);
|
|
2807
2942
|
}
|
|
2943
|
+
uncertain.push(...unsourced);
|
|
2808
2944
|
const blocked = parseBlocked(value.blocked);
|
|
2809
2945
|
return {
|
|
2810
2946
|
summary,
|
|
@@ -2923,6 +3059,227 @@ function renderUpdate(title, update, review) {
|
|
|
2923
3059
|
${note}`;
|
|
2924
3060
|
}
|
|
2925
3061
|
|
|
3062
|
+
// node_modules/@rackbops/docket-types/dist/want-sources.js
|
|
3063
|
+
import { createHash } from "crypto";
|
|
3064
|
+
class SourceMiss extends Error {
|
|
3065
|
+
name = "SourceMiss";
|
|
3066
|
+
}
|
|
3067
|
+
|
|
3068
|
+
class SourceUnavailableError extends SourceMiss {
|
|
3069
|
+
name = "SourceUnavailableError";
|
|
3070
|
+
}
|
|
3071
|
+
var MAX_LISTINGS = 100;
|
|
3072
|
+
var TITLE_CHARS = 150;
|
|
3073
|
+
var CONDITION_CHARS = 40;
|
|
3074
|
+
var SELLER_CHARS = 80;
|
|
3075
|
+
function checkSearchStatus(status, who, refused, unavailable = {}) {
|
|
3076
|
+
const said = unavailable[status] ?? (status === 401 || status === 403 ? refused : undefined);
|
|
3077
|
+
if (said !== undefined)
|
|
3078
|
+
throw new SourceUnavailableError(said);
|
|
3079
|
+
if (status === 429 || status >= 500)
|
|
3080
|
+
throw new SourceMiss(`${who} is busy (HTTP ${status})`);
|
|
3081
|
+
if (status !== 200)
|
|
3082
|
+
throw new SourceMiss(`${who} answered HTTP ${status}`);
|
|
3083
|
+
}
|
|
3084
|
+
var MAX_DEPTH = 8;
|
|
3085
|
+
function listingKey(id) {
|
|
3086
|
+
return createHash("sha256").update(id).digest("hex").slice(0, 32);
|
|
3087
|
+
}
|
|
3088
|
+
var NEVER_EBAY = "x-tracker-never-ebay";
|
|
3089
|
+
var EBAY_HOST = /(^|\.)ebay\.[a-z]{2,3}(\.[a-z]{2})?$/i;
|
|
3090
|
+
function isEbayHost(host) {
|
|
3091
|
+
return EBAY_HOST.test(host.replace(/\.$/, ""));
|
|
3092
|
+
}
|
|
3093
|
+
var isObj3 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
3094
|
+
function typesOf(node) {
|
|
3095
|
+
const t = node["@type"];
|
|
3096
|
+
const all = Array.isArray(t) ? t : [t];
|
|
3097
|
+
return all.filter((x) => typeof x === "string").map((x) => x.replace(/^https?:\/\/schema\.org\//, ""));
|
|
3098
|
+
}
|
|
3099
|
+
var PRODUCT_TYPES = new Set([
|
|
3100
|
+
"Product",
|
|
3101
|
+
"IndividualProduct",
|
|
3102
|
+
"ProductModel",
|
|
3103
|
+
"ProductGroup",
|
|
3104
|
+
"Book",
|
|
3105
|
+
"Game",
|
|
3106
|
+
"VideoGame",
|
|
3107
|
+
"Vehicle"
|
|
3108
|
+
]);
|
|
3109
|
+
function text(v, max) {
|
|
3110
|
+
if (typeof v === "string")
|
|
3111
|
+
return clean(v, max, true);
|
|
3112
|
+
if (isObj3(v) && typeof v.name === "string")
|
|
3113
|
+
return clean(v.name, max, true);
|
|
3114
|
+
return "";
|
|
3115
|
+
}
|
|
3116
|
+
function conditionOf(v) {
|
|
3117
|
+
const raw = text(v, 200);
|
|
3118
|
+
const m = /(New|Used|Refurbished|Damaged)Condition$/i.exec(raw);
|
|
3119
|
+
return m ? (m[1] ?? "").toLowerCase() : clean(raw, CONDITION_CHARS, true);
|
|
3120
|
+
}
|
|
3121
|
+
function firstOffer(offers) {
|
|
3122
|
+
if (Array.isArray(offers))
|
|
3123
|
+
return offers.find(isObj3) ?? null;
|
|
3124
|
+
return isObj3(offers) ? offers : null;
|
|
3125
|
+
}
|
|
3126
|
+
var PER_VIEW = /^(utm_[a-z]+|_pos|_sid|_ss|_psq|srsltid|gclid|gbraid|wbraid|fbclid|msclkid|mc_cid|mc_eid|pf_rd_[a-z]+|pd_rd_[a-z]+)$/i;
|
|
3127
|
+
function withoutPerView(u) {
|
|
3128
|
+
u.hash = "";
|
|
3129
|
+
for (const key of [...u.searchParams.keys()])
|
|
3130
|
+
if (PER_VIEW.test(key))
|
|
3131
|
+
u.searchParams.delete(key);
|
|
3132
|
+
return u;
|
|
3133
|
+
}
|
|
3134
|
+
function absolute(raw, base) {
|
|
3135
|
+
if (typeof raw !== "string" || raw.trim() === "")
|
|
3136
|
+
return null;
|
|
3137
|
+
try {
|
|
3138
|
+
return safeUrl(withoutPerView(new URL(raw.trim(), base)).toString());
|
|
3139
|
+
} catch {
|
|
3140
|
+
return null;
|
|
3141
|
+
}
|
|
3142
|
+
}
|
|
3143
|
+
function listingOf(node, base, fallbackUrl) {
|
|
3144
|
+
const title = text(node.name, TITLE_CHARS);
|
|
3145
|
+
const offer = firstOffer(node.offers);
|
|
3146
|
+
const url = absolute(node.url, base) ?? absolute(fallbackUrl, base) ?? absolute(offer?.url, base);
|
|
3147
|
+
if (!title || !url)
|
|
3148
|
+
return null;
|
|
3149
|
+
const price = priceFromJson(node.offers ?? null);
|
|
3150
|
+
const condition = conditionOf(offer?.itemCondition ?? node.itemCondition);
|
|
3151
|
+
const seller = text(offer?.seller, SELLER_CHARS);
|
|
3152
|
+
return {
|
|
3153
|
+
id: url,
|
|
3154
|
+
title,
|
|
3155
|
+
url,
|
|
3156
|
+
...price ? { price: price.value } : {},
|
|
3157
|
+
...price?.currency ? { currency: price.currency } : {},
|
|
3158
|
+
...condition ? { condition } : {},
|
|
3159
|
+
...seller ? { seller } : {}
|
|
3160
|
+
};
|
|
3161
|
+
}
|
|
3162
|
+
function readListings(html, pageUrl) {
|
|
3163
|
+
let lists = 0;
|
|
3164
|
+
const out = [];
|
|
3165
|
+
const seen = new Set;
|
|
3166
|
+
const add = (l) => {
|
|
3167
|
+
if (!l || seen.has(l.id) || out.length >= MAX_LISTINGS)
|
|
3168
|
+
return;
|
|
3169
|
+
seen.add(l.id);
|
|
3170
|
+
out.push(l);
|
|
3171
|
+
};
|
|
3172
|
+
const walk = (node, depth) => {
|
|
3173
|
+
if (depth > MAX_DEPTH || out.length >= MAX_LISTINGS)
|
|
3174
|
+
return;
|
|
3175
|
+
if (Array.isArray(node)) {
|
|
3176
|
+
for (const n of node)
|
|
3177
|
+
walk(n, depth + 1);
|
|
3178
|
+
return;
|
|
3179
|
+
}
|
|
3180
|
+
if (!isObj3(node))
|
|
3181
|
+
return;
|
|
3182
|
+
const types = typesOf(node);
|
|
3183
|
+
if (types.includes("ItemList")) {
|
|
3184
|
+
lists++;
|
|
3185
|
+
const elements = Array.isArray(node.itemListElement) ? node.itemListElement : [node.itemListElement];
|
|
3186
|
+
for (const el of elements) {
|
|
3187
|
+
if (!isObj3(el))
|
|
3188
|
+
continue;
|
|
3189
|
+
const item = isObj3(el.item) ? el.item : el;
|
|
3190
|
+
if (typesOf(item).includes("ItemList"))
|
|
3191
|
+
walk(item, depth + 1);
|
|
3192
|
+
else
|
|
3193
|
+
add(listingOf(item, pageUrl, el.url ?? (typeof el.item === "string" ? el.item : undefined)));
|
|
3194
|
+
}
|
|
3195
|
+
} else if (types.some((t) => PRODUCT_TYPES.has(t))) {
|
|
3196
|
+
add(listingOf(node, pageUrl, pageUrl));
|
|
3197
|
+
}
|
|
3198
|
+
if (node["@graph"] !== undefined)
|
|
3199
|
+
walk(node["@graph"], depth + 1);
|
|
3200
|
+
if (isObj3(node.mainEntity))
|
|
3201
|
+
walk(node.mainEntity, depth + 1);
|
|
3202
|
+
};
|
|
3203
|
+
for (const block of jsonLdBlocks(html))
|
|
3204
|
+
walk(block, 0);
|
|
3205
|
+
return { listings: out, lists };
|
|
3206
|
+
}
|
|
3207
|
+
var pageSource = {
|
|
3208
|
+
id: "page",
|
|
3209
|
+
async search(target, fetch2) {
|
|
3210
|
+
if (!fetch2)
|
|
3211
|
+
throw new SourceMiss("this bot reads no pages");
|
|
3212
|
+
let body;
|
|
3213
|
+
try {
|
|
3214
|
+
const response = await fetch2.get(target, { [NEVER_EBAY]: "1" });
|
|
3215
|
+
if (response.status !== 200)
|
|
3216
|
+
throw new SourceMiss(`HTTP ${response.status}`);
|
|
3217
|
+
body = response.body;
|
|
3218
|
+
} catch (err) {
|
|
3219
|
+
if (err instanceof SourceMiss)
|
|
3220
|
+
throw err;
|
|
3221
|
+
throw new SourceMiss(`the read failed: ${err instanceof Error ? err.message : String(err)}`, {
|
|
3222
|
+
cause: err
|
|
3223
|
+
});
|
|
3224
|
+
}
|
|
3225
|
+
const { listings, lists } = readListings(body, target);
|
|
3226
|
+
if (listings.length === 0 && lists === 0)
|
|
3227
|
+
throw new SourceMiss("no listings in the page's structured data");
|
|
3228
|
+
return listings;
|
|
3229
|
+
}
|
|
3230
|
+
};
|
|
3231
|
+
var nonNegative = (n) => n !== null && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
3232
|
+
function ebayItem(url) {
|
|
3233
|
+
if (url === null)
|
|
3234
|
+
return null;
|
|
3235
|
+
const u = new URL(url);
|
|
3236
|
+
const item = /^\/itm\/(?:[^/]+\/)?(\d{6,20})\/?$/.exec(u.pathname);
|
|
3237
|
+
return isEbayHost(u.hostname) && item ? `https://${u.hostname}/itm/${item[1]}` : url;
|
|
3238
|
+
}
|
|
3239
|
+
function submittedListing(raw) {
|
|
3240
|
+
const title = typeof raw.title === "string" ? clean(raw.title, TITLE_CHARS, true) : "";
|
|
3241
|
+
if (typeof raw.url !== "string" || !/^https?:\/\/[^/\\]/i.test(raw.url.trim()))
|
|
3242
|
+
return null;
|
|
3243
|
+
const url = ebayItem(absolute(raw.url, "https://invalid.example/"));
|
|
3244
|
+
if (!title || !url || url.startsWith("https://invalid.example/"))
|
|
3245
|
+
return null;
|
|
3246
|
+
const price = typeof raw.price === "number" && Number.isFinite(raw.price) && raw.price >= 0 ? raw.price : typeof raw.price === "string" ? nonNegative(parsePrice(raw.price)) : undefined;
|
|
3247
|
+
const currency = typeof raw.currency === "string" && /^[A-Za-z]{3}$/.test(raw.currency.trim()) ? raw.currency.trim().toUpperCase() : undefined;
|
|
3248
|
+
const condition = typeof raw.condition === "string" ? clean(raw.condition, CONDITION_CHARS, true) : "";
|
|
3249
|
+
const seller = typeof raw.seller === "string" ? clean(raw.seller, SELLER_CHARS, true) : "";
|
|
3250
|
+
return {
|
|
3251
|
+
id: url,
|
|
3252
|
+
title,
|
|
3253
|
+
url,
|
|
3254
|
+
...price !== undefined ? { price } : {},
|
|
3255
|
+
...currency ? { currency } : {},
|
|
3256
|
+
...condition ? { condition } : {},
|
|
3257
|
+
...seller ? { seller } : {}
|
|
3258
|
+
};
|
|
3259
|
+
}
|
|
3260
|
+
function inboxSource(read) {
|
|
3261
|
+
return {
|
|
3262
|
+
id: "inbox",
|
|
3263
|
+
async search(target) {
|
|
3264
|
+
let items;
|
|
3265
|
+
try {
|
|
3266
|
+
items = await read(target);
|
|
3267
|
+
} catch (err) {
|
|
3268
|
+
throw new SourceMiss("the inbox could not be read", { cause: err });
|
|
3269
|
+
}
|
|
3270
|
+
const byId = new Map;
|
|
3271
|
+
for (const raw of items) {
|
|
3272
|
+
const l = submittedListing(raw);
|
|
3273
|
+
if (!l)
|
|
3274
|
+
continue;
|
|
3275
|
+
byId.delete(l.id);
|
|
3276
|
+
byId.set(l.id, l);
|
|
3277
|
+
}
|
|
3278
|
+
return [...byId.values()].slice(-MAX_LISTINGS);
|
|
3279
|
+
}
|
|
3280
|
+
};
|
|
3281
|
+
}
|
|
3282
|
+
|
|
2926
3283
|
// node_modules/@rackbops/docket-types/dist/research-sources.js
|
|
2927
3284
|
var MAX_GATHERED_PAGES = 10;
|
|
2928
3285
|
var MAX_PAGE_TITLE_CHARS = 200;
|
|
@@ -2939,7 +3296,7 @@ var BARRED_SITES = [
|
|
|
2939
3296
|
{
|
|
2940
3297
|
site: "ebay.com",
|
|
2941
3298
|
domains: ["ebay.*", "ebay.co.*", "ebay.com.*"],
|
|
2942
|
-
host:
|
|
3299
|
+
host: EBAY_HOST,
|
|
2943
3300
|
names: "ebay",
|
|
2944
3301
|
everywhere: true
|
|
2945
3302
|
}
|
|
@@ -2975,13 +3332,13 @@ var GATHERED_DENIED_TOOLS = ["WebFetch"];
|
|
|
2975
3332
|
var WEB_DISALLOWED_TOOLS = barredDomains(true).map((host) => `WebFetch(domain:${host})`);
|
|
2976
3333
|
function hostOf(url) {
|
|
2977
3334
|
try {
|
|
2978
|
-
return new URL(url).hostname;
|
|
3335
|
+
return new URL(url).hostname.replace(/\.$/, "");
|
|
2979
3336
|
} catch {
|
|
2980
3337
|
return "";
|
|
2981
3338
|
}
|
|
2982
3339
|
}
|
|
2983
3340
|
function siteOf(text) {
|
|
2984
|
-
const t = text.trim().toLowerCase().replace(/^www\./, "");
|
|
3341
|
+
const t = text.trim().toLowerCase().replace(/^www\./, "").replace(/\.$/, "");
|
|
2985
3342
|
for (const b of BARRED_SITES)
|
|
2986
3343
|
if (b.host.test(t) || namedIn(b).test(t))
|
|
2987
3344
|
return b.site;
|
|
@@ -3154,27 +3511,36 @@ function categoryChecks() {
|
|
|
3154
3511
|
];
|
|
3155
3512
|
}
|
|
3156
3513
|
|
|
3514
|
+
// node_modules/@rackbops/docket-types/dist/retry.js
|
|
3515
|
+
var MAX_TRIES = 2;
|
|
3516
|
+
var AUTH_RETRY_MS = 3600000;
|
|
3517
|
+
var RETRIED = new Set([
|
|
3518
|
+
"schema_miss",
|
|
3519
|
+
"malformed",
|
|
3520
|
+
"timeout",
|
|
3521
|
+
"error",
|
|
3522
|
+
"auth_failed"
|
|
3523
|
+
]);
|
|
3524
|
+
function retryOutcome(kind, failures, now) {
|
|
3525
|
+
if (!RETRIED.has(kind) || failures >= MAX_TRIES)
|
|
3526
|
+
return null;
|
|
3527
|
+
const wait = kind === "auth_failed" ? AUTH_RETRY_MS : 0;
|
|
3528
|
+
const at = new Date(now.getTime() + wait).toISOString();
|
|
3529
|
+
return { at, followUp: wait > 0 ? { at } : {} };
|
|
3530
|
+
}
|
|
3531
|
+
|
|
3157
3532
|
// node_modules/@rackbops/docket-types/dist/research.js
|
|
3158
3533
|
function usesSentSources(config) {
|
|
3159
3534
|
return typeof config.sources === "string" && config.sources.trim().toLowerCase() === "sent";
|
|
3160
3535
|
}
|
|
3161
3536
|
var MAX_QUESTION_CHARS = 1000;
|
|
3162
3537
|
var MAX_CONTEXT_CHARS = 2000;
|
|
3163
|
-
var MAX_TRIES = 2;
|
|
3164
|
-
var AUTH_RETRY_MS = 3600000;
|
|
3165
3538
|
var RESEARCH_MAX_TURNS = 15;
|
|
3166
3539
|
var RESEARCH_MAX_BUDGET_USD = 1;
|
|
3167
3540
|
var RESEARCH_TIMEOUT_MS = 600000;
|
|
3168
3541
|
var REVIEW_MAX_TURNS = 8;
|
|
3169
3542
|
var REVIEW_MAX_BUDGET_USD = 0.5;
|
|
3170
3543
|
var REVIEW_TIMEOUT_MS = 300000;
|
|
3171
|
-
var RETRIED = new Set([
|
|
3172
|
-
"schema_miss",
|
|
3173
|
-
"malformed",
|
|
3174
|
-
"timeout",
|
|
3175
|
-
"error",
|
|
3176
|
-
"auth_failed"
|
|
3177
|
-
]);
|
|
3178
3544
|
var FRESH2 = { phase: "research", failures: 0, draft: null };
|
|
3179
3545
|
function passFields(s) {
|
|
3180
3546
|
const blocked = parseBlocked(s.blocked);
|
|
@@ -3392,14 +3758,9 @@ function failed(ctx, state, kind, detail) {
|
|
|
3392
3758
|
const phase = state.phase === "review" || state.phase === "awaiting-pages" ? state.phase : "research";
|
|
3393
3759
|
const failures = state.failures + 1;
|
|
3394
3760
|
const summary = clean(`${phase} ${kind} (try ${failures}): ${detail}`, 300, true);
|
|
3395
|
-
|
|
3396
|
-
|
|
3397
|
-
return {
|
|
3398
|
-
state: { ...state, phase, failures },
|
|
3399
|
-
followUp: wait > 0 ? { at: new Date(ctx.now.getTime() + wait).toISOString() } : {},
|
|
3400
|
-
summary
|
|
3401
|
-
};
|
|
3402
|
-
}
|
|
3761
|
+
const retry = retryOutcome(kind, failures, ctx.now);
|
|
3762
|
+
if (retry)
|
|
3763
|
+
return { state: { ...state, phase, failures }, followUp: retry.followUp, summary };
|
|
3403
3764
|
const said = SAID[kind] ?? "something went wrong";
|
|
3404
3765
|
if (secondPass(state)) {
|
|
3405
3766
|
return stands(ctx, state, `the fuller pass could not finish (${said})`, summary);
|
|
@@ -3685,7 +4046,7 @@ function researchType(options = {}) {
|
|
|
3685
4046
|
var research = researchType();
|
|
3686
4047
|
|
|
3687
4048
|
// node_modules/@rackbops/docket-types/dist/scout.js
|
|
3688
|
-
import { createHash } from "crypto";
|
|
4049
|
+
import { createHash as createHash2 } from "crypto";
|
|
3689
4050
|
var LENSES = ["general", "birthday", "anniversary", "christmas"];
|
|
3690
4051
|
var LENS_TEXT = {
|
|
3691
4052
|
general: "no occasion in particular -- anything they would genuinely enjoy hearing about or owning",
|
|
@@ -3734,18 +4095,17 @@ var SCOUT_SCHEMA = {
|
|
|
3734
4095
|
},
|
|
3735
4096
|
required: ["items"]
|
|
3736
4097
|
};
|
|
3737
|
-
function
|
|
4098
|
+
function isObj4(v) {
|
|
3738
4099
|
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
3739
4100
|
}
|
|
3740
4101
|
function itemKey(url) {
|
|
3741
|
-
const u = new URL(url);
|
|
3742
|
-
u.
|
|
3743
|
-
return createHash("sha256").update(u.toString()).digest("hex").slice(0, 32);
|
|
4102
|
+
const u = withoutPerView(new URL(url));
|
|
4103
|
+
return createHash2("sha256").update(u.toString()).digest("hex").slice(0, 32);
|
|
3744
4104
|
}
|
|
3745
4105
|
function scoutState(value) {
|
|
3746
|
-
if (!
|
|
4106
|
+
if (!isObj4(value))
|
|
3747
4107
|
return { shown: [], failures: 0 };
|
|
3748
|
-
const shown = (Array.isArray(value.shown) ? value.shown : []).filter((s) =>
|
|
4108
|
+
const shown = (Array.isArray(value.shown) ? value.shown : []).filter((s) => isObj4(s) && typeof s.k === "string" && typeof s.t === "string").slice(-MAX_SHOWN);
|
|
3749
4109
|
const failures = typeof value.failures === "number" && value.failures >= 0 ? value.failures : 0;
|
|
3750
4110
|
return { shown, failures };
|
|
3751
4111
|
}
|
|
@@ -3785,11 +4145,11 @@ function scoutJob(config, state) {
|
|
|
3785
4145
|
};
|
|
3786
4146
|
}
|
|
3787
4147
|
function parseScout(value) {
|
|
3788
|
-
if (!
|
|
4148
|
+
if (!isObj4(value) || !Array.isArray(value.items))
|
|
3789
4149
|
return null;
|
|
3790
4150
|
const items = [];
|
|
3791
4151
|
for (const raw of value.items) {
|
|
3792
|
-
if (!
|
|
4152
|
+
if (!isObj4(raw) || typeof raw.title !== "string")
|
|
3793
4153
|
continue;
|
|
3794
4154
|
const url = safeUrl(raw.url);
|
|
3795
4155
|
const title = clean(raw.title, ITEM_TITLE_CHARS, true);
|
|
@@ -3825,13 +4185,6 @@ function renderScout(title, items, shortfall, repeats) {
|
|
|
3825
4185
|
lines.push("", `Fewer than ${MIN_ITEMS}: ${shortfall}`);
|
|
3826
4186
|
return fitMessage(lines);
|
|
3827
4187
|
}
|
|
3828
|
-
var RETRIED2 = new Set([
|
|
3829
|
-
"schema_miss",
|
|
3830
|
-
"malformed",
|
|
3831
|
-
"timeout",
|
|
3832
|
-
"error",
|
|
3833
|
-
"auth_failed"
|
|
3834
|
-
]);
|
|
3835
4188
|
var SAID2 = {
|
|
3836
4189
|
auth_failed: "the runner could not sign in",
|
|
3837
4190
|
turn_cap: "it needed more steps than one run is allowed",
|
|
@@ -3844,14 +4197,9 @@ var SAID2 = {
|
|
|
3844
4197
|
function failed2(ctx, state, kind, detail) {
|
|
3845
4198
|
const failures = state.failures + 1;
|
|
3846
4199
|
const summary = clean(`scout ${kind} (try ${failures}): ${detail}`, 300, true);
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
return {
|
|
3850
|
-
state: { ...state, failures },
|
|
3851
|
-
followUp: wait > 0 ? { at: new Date(ctx.now.getTime() + wait).toISOString() } : {},
|
|
3852
|
-
summary
|
|
3853
|
-
};
|
|
3854
|
-
}
|
|
4200
|
+
const retry = retryOutcome(kind, failures, ctx.now);
|
|
4201
|
+
if (retry)
|
|
4202
|
+
return { state: { ...state, failures }, followUp: retry.followUp, summary };
|
|
3855
4203
|
const narrower = kind === "turn_cap" || kind === "budget_cap" ? " Fewer or narrower interests may help." : "";
|
|
3856
4204
|
return {
|
|
3857
4205
|
state: { ...state, failures: 0 },
|
|
@@ -3932,222 +4280,13 @@ var scout = defineTaskType({
|
|
|
3932
4280
|
},
|
|
3933
4281
|
async finish(ctx, result) {
|
|
3934
4282
|
const stored = scoutState(ctx.state);
|
|
3935
|
-
const state = ctx.occurrence.dedupeKey.startsWith(
|
|
4283
|
+
const state = ctx.occurrence.dedupeKey.startsWith(FOLLOW_UP_PREFIX) ? stored : { ...stored, failures: 0 };
|
|
3936
4284
|
if (result.kind !== "success")
|
|
3937
4285
|
return failed2(ctx, state, result.kind, result.detail);
|
|
3938
4286
|
return afterRun(ctx, state, result.structuredOutput);
|
|
3939
4287
|
}
|
|
3940
4288
|
});
|
|
3941
4289
|
|
|
3942
|
-
// node_modules/@rackbops/docket-types/dist/want-sources.js
|
|
3943
|
-
import { createHash as createHash2 } from "crypto";
|
|
3944
|
-
class SourceMiss extends Error {
|
|
3945
|
-
name = "SourceMiss";
|
|
3946
|
-
}
|
|
3947
|
-
|
|
3948
|
-
class SourceUnavailableError extends SourceMiss {
|
|
3949
|
-
name = "SourceUnavailableError";
|
|
3950
|
-
}
|
|
3951
|
-
var MAX_LISTINGS = 100;
|
|
3952
|
-
var TITLE_CHARS = 150;
|
|
3953
|
-
var CONDITION_CHARS = 40;
|
|
3954
|
-
var SELLER_CHARS = 80;
|
|
3955
|
-
var MAX_DEPTH = 8;
|
|
3956
|
-
function listingKey(id) {
|
|
3957
|
-
return createHash2("sha256").update(id).digest("hex").slice(0, 32);
|
|
3958
|
-
}
|
|
3959
|
-
var NEVER_EBAY = "x-tracker-never-ebay";
|
|
3960
|
-
function isEbayHost(host) {
|
|
3961
|
-
return /(^|\.)ebay\.[a-z]{2,3}(\.[a-z]{2})?$/i.test(host.replace(/\.$/, ""));
|
|
3962
|
-
}
|
|
3963
|
-
var isObj4 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
3964
|
-
function typesOf(node) {
|
|
3965
|
-
const t = node["@type"];
|
|
3966
|
-
const all = Array.isArray(t) ? t : [t];
|
|
3967
|
-
return all.filter((x) => typeof x === "string").map((x) => x.replace(/^https?:\/\/schema\.org\//, ""));
|
|
3968
|
-
}
|
|
3969
|
-
var PRODUCT_TYPES = new Set([
|
|
3970
|
-
"Product",
|
|
3971
|
-
"IndividualProduct",
|
|
3972
|
-
"ProductModel",
|
|
3973
|
-
"ProductGroup",
|
|
3974
|
-
"Book",
|
|
3975
|
-
"Game",
|
|
3976
|
-
"VideoGame",
|
|
3977
|
-
"Vehicle"
|
|
3978
|
-
]);
|
|
3979
|
-
function text(v, max) {
|
|
3980
|
-
if (typeof v === "string")
|
|
3981
|
-
return clean(v, max, true);
|
|
3982
|
-
if (isObj4(v) && typeof v.name === "string")
|
|
3983
|
-
return clean(v.name, max, true);
|
|
3984
|
-
return "";
|
|
3985
|
-
}
|
|
3986
|
-
function conditionOf(v) {
|
|
3987
|
-
const raw = text(v, 200);
|
|
3988
|
-
const m = /(New|Used|Refurbished|Damaged)Condition$/i.exec(raw);
|
|
3989
|
-
return m ? (m[1] ?? "").toLowerCase() : clean(raw, CONDITION_CHARS, true);
|
|
3990
|
-
}
|
|
3991
|
-
function firstOffer(offers) {
|
|
3992
|
-
if (Array.isArray(offers))
|
|
3993
|
-
return offers.find(isObj4) ?? null;
|
|
3994
|
-
return isObj4(offers) ? offers : null;
|
|
3995
|
-
}
|
|
3996
|
-
var PER_VIEW = /^(utm_[a-z]+|_pos|_sid|_ss|_psq|srsltid|gclid|gbraid|wbraid|fbclid|msclkid|mc_cid|mc_eid|pf_rd_[a-z]+|pd_rd_[a-z]+)$/i;
|
|
3997
|
-
function absolute(raw, base) {
|
|
3998
|
-
if (typeof raw !== "string" || raw.trim() === "")
|
|
3999
|
-
return null;
|
|
4000
|
-
try {
|
|
4001
|
-
const u = new URL(raw.trim(), base);
|
|
4002
|
-
u.hash = "";
|
|
4003
|
-
for (const key of [...u.searchParams.keys()])
|
|
4004
|
-
if (PER_VIEW.test(key))
|
|
4005
|
-
u.searchParams.delete(key);
|
|
4006
|
-
return safeUrl(u.toString());
|
|
4007
|
-
} catch {
|
|
4008
|
-
return null;
|
|
4009
|
-
}
|
|
4010
|
-
}
|
|
4011
|
-
function listingOf(node, base, fallbackUrl) {
|
|
4012
|
-
const title = text(node.name, TITLE_CHARS);
|
|
4013
|
-
const offer = firstOffer(node.offers);
|
|
4014
|
-
const url = absolute(node.url, base) ?? absolute(fallbackUrl, base) ?? absolute(offer?.url, base);
|
|
4015
|
-
if (!title || !url)
|
|
4016
|
-
return null;
|
|
4017
|
-
const price = priceFromJson(node.offers ?? null);
|
|
4018
|
-
const condition = conditionOf(offer?.itemCondition ?? node.itemCondition);
|
|
4019
|
-
const seller = text(offer?.seller, SELLER_CHARS);
|
|
4020
|
-
return {
|
|
4021
|
-
id: url,
|
|
4022
|
-
title,
|
|
4023
|
-
url,
|
|
4024
|
-
...price ? { price: price.value } : {},
|
|
4025
|
-
...price?.currency ? { currency: price.currency } : {},
|
|
4026
|
-
...condition ? { condition } : {},
|
|
4027
|
-
...seller ? { seller } : {}
|
|
4028
|
-
};
|
|
4029
|
-
}
|
|
4030
|
-
function readListings(html, pageUrl) {
|
|
4031
|
-
let lists = 0;
|
|
4032
|
-
const out = [];
|
|
4033
|
-
const seen = new Set;
|
|
4034
|
-
const add = (l) => {
|
|
4035
|
-
if (!l || seen.has(l.id) || out.length >= MAX_LISTINGS)
|
|
4036
|
-
return;
|
|
4037
|
-
seen.add(l.id);
|
|
4038
|
-
out.push(l);
|
|
4039
|
-
};
|
|
4040
|
-
const walk = (node, depth) => {
|
|
4041
|
-
if (depth > MAX_DEPTH || out.length >= MAX_LISTINGS)
|
|
4042
|
-
return;
|
|
4043
|
-
if (Array.isArray(node)) {
|
|
4044
|
-
for (const n of node)
|
|
4045
|
-
walk(n, depth + 1);
|
|
4046
|
-
return;
|
|
4047
|
-
}
|
|
4048
|
-
if (!isObj4(node))
|
|
4049
|
-
return;
|
|
4050
|
-
const types = typesOf(node);
|
|
4051
|
-
if (types.includes("ItemList")) {
|
|
4052
|
-
lists++;
|
|
4053
|
-
const elements = Array.isArray(node.itemListElement) ? node.itemListElement : [node.itemListElement];
|
|
4054
|
-
for (const el of elements) {
|
|
4055
|
-
if (!isObj4(el))
|
|
4056
|
-
continue;
|
|
4057
|
-
const item = isObj4(el.item) ? el.item : el;
|
|
4058
|
-
if (typesOf(item).includes("ItemList"))
|
|
4059
|
-
walk(item, depth + 1);
|
|
4060
|
-
else
|
|
4061
|
-
add(listingOf(item, pageUrl, el.url ?? (typeof el.item === "string" ? el.item : undefined)));
|
|
4062
|
-
}
|
|
4063
|
-
} else if (types.some((t) => PRODUCT_TYPES.has(t))) {
|
|
4064
|
-
add(listingOf(node, pageUrl, pageUrl));
|
|
4065
|
-
}
|
|
4066
|
-
if (node["@graph"] !== undefined)
|
|
4067
|
-
walk(node["@graph"], depth + 1);
|
|
4068
|
-
if (isObj4(node.mainEntity))
|
|
4069
|
-
walk(node.mainEntity, depth + 1);
|
|
4070
|
-
};
|
|
4071
|
-
for (const block of jsonLdBlocks(html))
|
|
4072
|
-
walk(block, 0);
|
|
4073
|
-
return { listings: out, lists };
|
|
4074
|
-
}
|
|
4075
|
-
var pageSource = {
|
|
4076
|
-
id: "page",
|
|
4077
|
-
async search(target, fetch2) {
|
|
4078
|
-
if (!fetch2)
|
|
4079
|
-
throw new SourceMiss("this bot reads no pages");
|
|
4080
|
-
let body;
|
|
4081
|
-
try {
|
|
4082
|
-
const response = await fetch2.get(target, { [NEVER_EBAY]: "1" });
|
|
4083
|
-
if (response.status !== 200)
|
|
4084
|
-
throw new SourceMiss(`HTTP ${response.status}`);
|
|
4085
|
-
body = response.body;
|
|
4086
|
-
} catch (err) {
|
|
4087
|
-
if (err instanceof SourceMiss)
|
|
4088
|
-
throw err;
|
|
4089
|
-
throw new SourceMiss(`the read failed: ${err instanceof Error ? err.message : String(err)}`, {
|
|
4090
|
-
cause: err
|
|
4091
|
-
});
|
|
4092
|
-
}
|
|
4093
|
-
const { listings, lists } = readListings(body, target);
|
|
4094
|
-
if (listings.length === 0 && lists === 0)
|
|
4095
|
-
throw new SourceMiss("no listings in the page's structured data");
|
|
4096
|
-
return listings;
|
|
4097
|
-
}
|
|
4098
|
-
};
|
|
4099
|
-
var nonNegative = (n) => n !== null && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
4100
|
-
function ebayItem(url) {
|
|
4101
|
-
if (url === null)
|
|
4102
|
-
return null;
|
|
4103
|
-
const u = new URL(url);
|
|
4104
|
-
const item = /^\/itm\/(?:[^/]+\/)?(\d{6,20})\/?$/.exec(u.pathname);
|
|
4105
|
-
return isEbayHost(u.hostname) && item ? `https://${u.hostname}/itm/${item[1]}` : url;
|
|
4106
|
-
}
|
|
4107
|
-
function submittedListing(raw) {
|
|
4108
|
-
const title = typeof raw.title === "string" ? clean(raw.title, TITLE_CHARS, true) : "";
|
|
4109
|
-
if (typeof raw.url !== "string" || !/^https?:\/\/[^/\\]/i.test(raw.url.trim()))
|
|
4110
|
-
return null;
|
|
4111
|
-
const url = ebayItem(absolute(raw.url, "https://invalid.example/"));
|
|
4112
|
-
if (!title || !url || url.startsWith("https://invalid.example/"))
|
|
4113
|
-
return null;
|
|
4114
|
-
const price = typeof raw.price === "number" && Number.isFinite(raw.price) && raw.price >= 0 ? raw.price : typeof raw.price === "string" ? nonNegative(parsePrice(raw.price)) : undefined;
|
|
4115
|
-
const currency = typeof raw.currency === "string" && /^[A-Za-z]{3}$/.test(raw.currency.trim()) ? raw.currency.trim().toUpperCase() : undefined;
|
|
4116
|
-
const condition = typeof raw.condition === "string" ? clean(raw.condition, CONDITION_CHARS, true) : "";
|
|
4117
|
-
const seller = typeof raw.seller === "string" ? clean(raw.seller, SELLER_CHARS, true) : "";
|
|
4118
|
-
return {
|
|
4119
|
-
id: url,
|
|
4120
|
-
title,
|
|
4121
|
-
url,
|
|
4122
|
-
...price !== undefined ? { price } : {},
|
|
4123
|
-
...currency ? { currency } : {},
|
|
4124
|
-
...condition ? { condition } : {},
|
|
4125
|
-
...seller ? { seller } : {}
|
|
4126
|
-
};
|
|
4127
|
-
}
|
|
4128
|
-
function inboxSource(read) {
|
|
4129
|
-
return {
|
|
4130
|
-
id: "inbox",
|
|
4131
|
-
async search(target) {
|
|
4132
|
-
let items;
|
|
4133
|
-
try {
|
|
4134
|
-
items = await read(target);
|
|
4135
|
-
} catch (err) {
|
|
4136
|
-
throw new SourceMiss("the inbox could not be read", { cause: err });
|
|
4137
|
-
}
|
|
4138
|
-
const byId = new Map;
|
|
4139
|
-
for (const raw of items) {
|
|
4140
|
-
const l = submittedListing(raw);
|
|
4141
|
-
if (!l)
|
|
4142
|
-
continue;
|
|
4143
|
-
byId.delete(l.id);
|
|
4144
|
-
byId.set(l.id, l);
|
|
4145
|
-
}
|
|
4146
|
-
return [...byId.values()].slice(-MAX_LISTINGS);
|
|
4147
|
-
}
|
|
4148
|
-
};
|
|
4149
|
-
}
|
|
4150
|
-
|
|
4151
4290
|
// node_modules/@rackbops/docket-types/dist/want-bgg.js
|
|
4152
4291
|
var BGG_HOST = "boardgamegeek.com";
|
|
4153
4292
|
var BGG_SPACING_MS = 5000;
|
|
@@ -4307,11 +4446,8 @@ var SHOPIFY_COUNTRY = "US";
|
|
|
4307
4446
|
var SHOPIFY_CURRENCY = "USD";
|
|
4308
4447
|
var SHOPIFY_ATTRIBUTION = "via Shopify";
|
|
4309
4448
|
var MAX_VARIANTS = 20;
|
|
4310
|
-
var TITLE_CHARS2 = 150;
|
|
4311
|
-
var SELLER_CHARS2 = 80;
|
|
4312
4449
|
function shopifyRequest(s) {
|
|
4313
|
-
const
|
|
4314
|
-
const currency = /^[A-Z]{3}$/.test(asked) ? asked : SHOPIFY_CURRENCY;
|
|
4450
|
+
const currency = currencyCode(s.currency) ?? SHOPIFY_CURRENCY;
|
|
4315
4451
|
const filters = { available: true };
|
|
4316
4452
|
if (s.maxPrice !== undefined && Number.isFinite(s.maxPrice) && s.maxPrice >= 0)
|
|
4317
4453
|
filters.price = { max: Math.floor(s.maxPrice * minorUnits(currency) + 0.000001) };
|
|
@@ -4327,7 +4463,6 @@ function shopifyRequest(s) {
|
|
|
4327
4463
|
}
|
|
4328
4464
|
};
|
|
4329
4465
|
}
|
|
4330
|
-
var isObj5 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4331
4466
|
var QUERY_CHARS = 200;
|
|
4332
4467
|
function shopifyQuery(query) {
|
|
4333
4468
|
const line = clean(query, 1e4, true);
|
|
@@ -4364,14 +4499,8 @@ function listingUrl(raw) {
|
|
|
4364
4499
|
u.searchParams.delete(key);
|
|
4365
4500
|
return safeUrl(u.toString());
|
|
4366
4501
|
}
|
|
4367
|
-
var ZERO_DECIMAL = new Set("BIF CLP DJF GNF ISK JPY KMF KRW PYG RWF UGX UYI VND VUV XAF XOF XPF".split(" "));
|
|
4368
|
-
var THREE_DECIMAL = new Set("BHD IQD JOD KWD LYD OMR TND".split(" "));
|
|
4369
|
-
function minorUnits(currency) {
|
|
4370
|
-
const c = currency.toUpperCase();
|
|
4371
|
-
return ZERO_DECIMAL.has(c) ? 1 : THREE_DECIMAL.has(c) ? 1000 : 100;
|
|
4372
|
-
}
|
|
4373
4502
|
function priceOf(v) {
|
|
4374
|
-
if (!
|
|
4503
|
+
if (!isObj3(v))
|
|
4375
4504
|
return null;
|
|
4376
4505
|
const amount = v.amount;
|
|
4377
4506
|
const currency = typeof v.currency === "string" ? v.currency.trim().toUpperCase() : "";
|
|
@@ -4389,17 +4518,17 @@ function isNew(variant) {
|
|
|
4389
4518
|
return all.every((x) => typeof x === "string" && x.trim().toLowerCase() === "new");
|
|
4390
4519
|
}
|
|
4391
4520
|
function parseShopifyAnswer(body, query) {
|
|
4392
|
-
const result =
|
|
4521
|
+
const result = isObj3(body) && isObj3(body.data) ? body.data.alexandria : undefined;
|
|
4393
4522
|
const first = Array.isArray(result) ? result[0] : undefined;
|
|
4394
|
-
if (!
|
|
4523
|
+
if (!isObj3(first))
|
|
4395
4524
|
throw new SourceMiss("Firecrawl's answer had no search result");
|
|
4396
4525
|
if (first.error !== undefined && first.error !== null) {
|
|
4397
4526
|
const e = first.error;
|
|
4398
|
-
const why =
|
|
4527
|
+
const why = isObj3(e) && typeof e.code === "string" ? clean(e.code, 60, true) : "an error";
|
|
4399
4528
|
throw new SourceMiss(`Shopify's search failed (${why})`);
|
|
4400
4529
|
}
|
|
4401
4530
|
const data = first.data;
|
|
4402
|
-
if (!
|
|
4531
|
+
if (!isObj3(data) || !Array.isArray(data.products))
|
|
4403
4532
|
throw new SourceMiss("Shopify's search answer had no products");
|
|
4404
4533
|
const words = searchWords(query);
|
|
4405
4534
|
const out = [];
|
|
@@ -4407,32 +4536,32 @@ function parseShopifyAnswer(body, query) {
|
|
|
4407
4536
|
for (const product of data.products.slice(0, SHOPIFY_LIMIT)) {
|
|
4408
4537
|
if (out.length >= MAX_LISTINGS)
|
|
4409
4538
|
break;
|
|
4410
|
-
if (!
|
|
4539
|
+
if (!isObj3(product) || typeof product.title !== "string")
|
|
4411
4540
|
continue;
|
|
4412
4541
|
if (!matchesWords(product.title.slice(0, 2000), words))
|
|
4413
4542
|
continue;
|
|
4414
|
-
const title = clean(product.title,
|
|
4543
|
+
const title = clean(product.title, TITLE_CHARS, true);
|
|
4415
4544
|
if (!title)
|
|
4416
4545
|
continue;
|
|
4417
4546
|
const variants = Array.isArray(product.variants) ? product.variants.slice(0, MAX_VARIANTS) : [];
|
|
4418
4547
|
for (const v of variants) {
|
|
4419
4548
|
if (out.length >= MAX_LISTINGS)
|
|
4420
4549
|
break;
|
|
4421
|
-
if (!
|
|
4550
|
+
if (!isObj3(v) || typeof v.id !== "string" || v.id.length > 300 || !isNew(v))
|
|
4422
4551
|
continue;
|
|
4423
|
-
if (!
|
|
4552
|
+
if (!isObj3(v.availability) || v.availability.available !== true)
|
|
4424
4553
|
continue;
|
|
4425
4554
|
const url = listingUrl(v.url);
|
|
4426
4555
|
const id = `shopify:${v.id}`;
|
|
4427
4556
|
if (!url || seen.has(id))
|
|
4428
4557
|
continue;
|
|
4429
4558
|
seen.add(id);
|
|
4430
|
-
const own = typeof v.title === "string" ? clean(v.title,
|
|
4559
|
+
const own = typeof v.title === "string" ? clean(v.title, TITLE_CHARS, true) : "";
|
|
4431
4560
|
const named = own !== "" && own.toLowerCase() !== "default title" && !title.toLowerCase().includes(own.toLowerCase());
|
|
4432
|
-
const name = named ? clean(`${title} (${own})`,
|
|
4561
|
+
const name = named ? clean(`${title} (${own})`, TITLE_CHARS, true) : title;
|
|
4433
4562
|
const price = priceOf(v.price);
|
|
4434
|
-
const seller =
|
|
4435
|
-
const sold = clean(seller,
|
|
4563
|
+
const seller = isObj3(v.seller) && typeof v.seller.name === "string" ? v.seller.name : "";
|
|
4564
|
+
const sold = clean(seller, SELLER_CHARS, true);
|
|
4436
4565
|
out.push({
|
|
4437
4566
|
id,
|
|
4438
4567
|
title: name,
|
|
@@ -4458,12 +4587,9 @@ function shopifySource(scrape) {
|
|
|
4458
4587
|
} catch (err) {
|
|
4459
4588
|
throw new SourceMiss(`Shopify could not be searched: ${clean(err instanceof Error ? err.message : String(err), 200, true)}`, { cause: err });
|
|
4460
4589
|
}
|
|
4461
|
-
|
|
4462
|
-
|
|
4463
|
-
|
|
4464
|
-
throw new SourceMiss(`Firecrawl is busy (HTTP ${status})`);
|
|
4465
|
-
if (status !== 200)
|
|
4466
|
-
throw new SourceMiss(`Firecrawl answered HTTP ${status}`);
|
|
4590
|
+
checkSearchStatus(status, "Firecrawl", "Firecrawl refused the key", {
|
|
4591
|
+
402: "the Firecrawl account is out of credits"
|
|
4592
|
+
});
|
|
4467
4593
|
return parseShopifyAnswer(body, target);
|
|
4468
4594
|
}
|
|
4469
4595
|
};
|
|
@@ -4475,12 +4601,8 @@ var EBAY_MARKETPLACE = "EBAY_US";
|
|
|
4475
4601
|
var EBAY_CURRENCY = "USD";
|
|
4476
4602
|
var EBAY_LIMIT = 50;
|
|
4477
4603
|
var EBAY_ATTRIBUTION = "via eBay";
|
|
4478
|
-
var TITLE_CHARS3 = 150;
|
|
4479
|
-
var CONDITION_CHARS2 = 40;
|
|
4480
|
-
var SELLER_CHARS3 = 80;
|
|
4481
4604
|
function ebayRequest(s) {
|
|
4482
|
-
const
|
|
4483
|
-
const currency = asked === "" ? EBAY_CURRENCY : asked;
|
|
4605
|
+
const currency = currencyCode(s.currency) ?? EBAY_CURRENCY;
|
|
4484
4606
|
const params = {
|
|
4485
4607
|
q: shopifyQuery(s.query),
|
|
4486
4608
|
limit: String(EBAY_LIMIT),
|
|
@@ -4492,9 +4614,8 @@ function ebayRequest(s) {
|
|
|
4492
4614
|
}
|
|
4493
4615
|
return { path: EBAY_SEARCH_PATH, params, marketplace: EBAY_MARKETPLACE };
|
|
4494
4616
|
}
|
|
4495
|
-
var isObj6 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4496
4617
|
function amountOf(v) {
|
|
4497
|
-
if (!
|
|
4618
|
+
if (!isObj3(v))
|
|
4498
4619
|
return null;
|
|
4499
4620
|
const raw = typeof v.value === "string" || typeof v.value === "number" ? String(v.value) : "";
|
|
4500
4621
|
const currency = typeof v.currency === "string" ? v.currency.trim().toUpperCase() : "";
|
|
@@ -4517,7 +4638,7 @@ function itemUrl(raw) {
|
|
|
4517
4638
|
return item ? safeUrl(`https://${u.hostname.toLowerCase()}/itm/${item[1]}`) : null;
|
|
4518
4639
|
}
|
|
4519
4640
|
function sellerOf(v) {
|
|
4520
|
-
if (!
|
|
4641
|
+
if (!isObj3(v) || typeof v.username !== "string")
|
|
4521
4642
|
return "";
|
|
4522
4643
|
const name = clean(v.username, 40, true);
|
|
4523
4644
|
if (!name)
|
|
@@ -4528,10 +4649,10 @@ function sellerOf(v) {
|
|
|
4528
4649
|
parts.push(`${pct}% positive`);
|
|
4529
4650
|
if (typeof v.feedbackScore === "number" && Number.isInteger(v.feedbackScore))
|
|
4530
4651
|
parts.push(`${v.feedbackScore} feedback`);
|
|
4531
|
-
return clean(parts.join(", "),
|
|
4652
|
+
return clean(parts.join(", "), SELLER_CHARS, true);
|
|
4532
4653
|
}
|
|
4533
4654
|
function parseEbayAnswer(body, query) {
|
|
4534
|
-
if (!
|
|
4655
|
+
if (!isObj3(body) || body.itemSummaries === undefined && typeof body.total !== "number")
|
|
4535
4656
|
throw new SourceMiss("eBay's answer was not a search result");
|
|
4536
4657
|
const items = Array.isArray(body.itemSummaries) ? body.itemSummaries : [];
|
|
4537
4658
|
const words = searchWords(query);
|
|
@@ -4540,11 +4661,11 @@ function parseEbayAnswer(body, query) {
|
|
|
4540
4661
|
for (const item of items.slice(0, EBAY_LIMIT * 4)) {
|
|
4541
4662
|
if (out.length >= MAX_LISTINGS)
|
|
4542
4663
|
break;
|
|
4543
|
-
if (!
|
|
4664
|
+
if (!isObj3(item) || typeof item.title !== "string" || typeof item.itemId !== "string")
|
|
4544
4665
|
continue;
|
|
4545
4666
|
if (!matchesWords(item.title.slice(0, 2000), words))
|
|
4546
4667
|
continue;
|
|
4547
|
-
const title = clean(item.title,
|
|
4668
|
+
const title = clean(item.title, TITLE_CHARS, true);
|
|
4548
4669
|
const url = itemUrl(item.itemWebUrl);
|
|
4549
4670
|
if (!title || !url || item.itemId.length > 120)
|
|
4550
4671
|
continue;
|
|
@@ -4558,7 +4679,7 @@ function parseEbayAnswer(body, query) {
|
|
|
4558
4679
|
const price = amountOf(auction && !buyNow ? item.currentBidPrice ?? item.price : item.price);
|
|
4559
4680
|
const state = typeof item.condition === "string" ? clean(item.condition, 30, true) : "";
|
|
4560
4681
|
const kind = auction ? [buyNow ? "auction, Buy It Now" : "auction"] : [];
|
|
4561
|
-
const condition = clean([state, ...kind].filter(Boolean).join(", "),
|
|
4682
|
+
const condition = clean([state, ...kind].filter(Boolean).join(", "), CONDITION_CHARS, true);
|
|
4562
4683
|
const seller = sellerOf(item.seller);
|
|
4563
4684
|
out.push({
|
|
4564
4685
|
id: `ebay:${number}`,
|
|
@@ -4584,17 +4705,49 @@ function ebaySource(browse) {
|
|
|
4584
4705
|
} catch (err) {
|
|
4585
4706
|
throw new SourceMiss(`eBay could not be searched: ${clean(err instanceof Error ? err.message : String(err), 200, true)}`, { cause: err });
|
|
4586
4707
|
}
|
|
4587
|
-
|
|
4588
|
-
throw new SourceUnavailableError("eBay refused the app's keys");
|
|
4589
|
-
if (status === 429 || status >= 500)
|
|
4590
|
-
throw new SourceMiss(`eBay is busy (HTTP ${status})`);
|
|
4591
|
-
if (status !== 200)
|
|
4592
|
-
throw new SourceMiss(`eBay answered HTTP ${status}`);
|
|
4708
|
+
checkSearchStatus(status, "eBay", "eBay refused the app's keys");
|
|
4593
4709
|
return parseEbayAnswer(body, target);
|
|
4594
4710
|
}
|
|
4595
4711
|
};
|
|
4596
4712
|
}
|
|
4597
4713
|
|
|
4714
|
+
// node_modules/@rackbops/docket-types/dist/want-text.js
|
|
4715
|
+
var SOURCE_TEXT = {
|
|
4716
|
+
page: {
|
|
4717
|
+
attribution: null,
|
|
4718
|
+
where: null,
|
|
4719
|
+
missing: "such source",
|
|
4720
|
+
judgeWhere: "a shop page they gave"
|
|
4721
|
+
},
|
|
4722
|
+
bgg: {
|
|
4723
|
+
attribution: BGG_ATTRIBUTION,
|
|
4724
|
+
where: "BoardGameGeek",
|
|
4725
|
+
missing: "BoardGameGeek access",
|
|
4726
|
+
judgeWhere: "BoardGameGeek's marketplace"
|
|
4727
|
+
},
|
|
4728
|
+
inbox: {
|
|
4729
|
+
attribution: null,
|
|
4730
|
+
where: "what was sent in",
|
|
4731
|
+
missing: "such source",
|
|
4732
|
+
judgeWhere: "alert emails and searches sent in for them (from eBay, BoardGameGeek or a shop)"
|
|
4733
|
+
},
|
|
4734
|
+
shopify: {
|
|
4735
|
+
attribution: SHOPIFY_ATTRIBUTION,
|
|
4736
|
+
where: "Shopify's store search",
|
|
4737
|
+
missing: "Shopify search (no Firecrawl key)",
|
|
4738
|
+
judgeWhere: "a search of Shopify stores' catalogs (new copies in stock)"
|
|
4739
|
+
},
|
|
4740
|
+
ebay: {
|
|
4741
|
+
attribution: EBAY_ATTRIBUTION,
|
|
4742
|
+
where: "eBay's search",
|
|
4743
|
+
missing: "eBay search (no eBay keys)",
|
|
4744
|
+
judgeWhere: "a search of eBay's listings, through eBay's own API"
|
|
4745
|
+
}
|
|
4746
|
+
};
|
|
4747
|
+
function sourceText(source) {
|
|
4748
|
+
return Object.hasOwn(SOURCE_TEXT, source) ? SOURCE_TEXT[source] : SOURCE_TEXT.page;
|
|
4749
|
+
}
|
|
4750
|
+
|
|
4598
4751
|
// node_modules/@rackbops/docket-types/dist/wantlist.js
|
|
4599
4752
|
var MAX_REPORTED = 500;
|
|
4600
4753
|
var SHOWN_IN_DM = 5;
|
|
@@ -4624,8 +4777,10 @@ function wantState(value) {
|
|
|
4624
4777
|
};
|
|
4625
4778
|
}
|
|
4626
4779
|
function withinLimits(l, config) {
|
|
4627
|
-
|
|
4628
|
-
|
|
4780
|
+
if (l.price !== undefined && !(l.price >= 0))
|
|
4781
|
+
return false;
|
|
4782
|
+
const want = currencyCode(config.currency);
|
|
4783
|
+
if (want && l.currency && currencyCode(l.currency) !== want)
|
|
4629
4784
|
return false;
|
|
4630
4785
|
if (config.maxPrice === undefined)
|
|
4631
4786
|
return true;
|
|
@@ -4641,7 +4796,8 @@ function listingLine(l, config) {
|
|
|
4641
4796
|
...l.condition ? [inert(l.condition)] : [],
|
|
4642
4797
|
...l.seller ? [`sold by ${inert(l.seller)}`] : []
|
|
4643
4798
|
];
|
|
4644
|
-
const
|
|
4799
|
+
const attribution = sourceText(config.source).attribution;
|
|
4800
|
+
const via = attribution ? ` (${attribution})` : "";
|
|
4645
4801
|
return `- ${parts.join(" -- ")} <${l.url}>${via}`;
|
|
4646
4802
|
}
|
|
4647
4803
|
function renderWant(title, taskId, fresh, config) {
|
|
@@ -4665,7 +4821,7 @@ function miss2(ctx, state, err) {
|
|
|
4665
4821
|
summary: `nothing read: ${reason}`
|
|
4666
4822
|
};
|
|
4667
4823
|
if (tell) {
|
|
4668
|
-
const where = ctx.config.source
|
|
4824
|
+
const where = sourceText(ctx.config.source).where ?? ctx.config.target;
|
|
4669
4825
|
outcome.notify = {
|
|
4670
4826
|
text: clip(`${ctx.task.title}: I could read no listings from ${where} ${misses === 1 ? "just now" : `${misses} times in a row`} (${reason}). ` + "I keep looking; press Done to stop."),
|
|
4671
4827
|
actions: ["done"]
|
|
@@ -4687,7 +4843,14 @@ async function pollWant(ctx, sources, stored) {
|
|
|
4687
4843
|
if (!source) {
|
|
4688
4844
|
return {
|
|
4689
4845
|
kind: "done",
|
|
4690
|
-
outcome: miss2(ctx, state, new SourceUnavailableError(`this bot has no ${ctx.config.source
|
|
4846
|
+
outcome: miss2(ctx, state, new SourceUnavailableError(`this bot has no ${sourceText(ctx.config.source).missing}`))
|
|
4847
|
+
};
|
|
4848
|
+
}
|
|
4849
|
+
const currency = currencyCode(ctx.config.currency);
|
|
4850
|
+
if (ctx.config.source === "ebay" && currency && currency !== EBAY_CURRENCY) {
|
|
4851
|
+
return {
|
|
4852
|
+
kind: "done",
|
|
4853
|
+
outcome: miss2(ctx, state, new SourceUnavailableError(`eBay US lists prices in ${EBAY_CURRENCY} only, so a watch in ${currency} never matches; ` + `set the currency to ${EBAY_CURRENCY} or leave it out`))
|
|
4691
4854
|
};
|
|
4692
4855
|
}
|
|
4693
4856
|
let listings;
|
|
@@ -4778,7 +4941,7 @@ var JUDGE_MAX_BUDGET_USD = 0.5;
|
|
|
4778
4941
|
var JUDGE_TIMEOUT_MS = 300000;
|
|
4779
4942
|
var RETRY_GRACE_MS = 30 * 60000;
|
|
4780
4943
|
var WHY_CHARS = 200;
|
|
4781
|
-
var
|
|
4944
|
+
var SELLER_CHARS2 = 200;
|
|
4782
4945
|
var PENDING_TITLE = 150;
|
|
4783
4946
|
var PENDING_TEXT = 80;
|
|
4784
4947
|
var JUDGE_SCHEMA = {
|
|
@@ -4801,10 +4964,9 @@ var JUDGE_SCHEMA = {
|
|
|
4801
4964
|
},
|
|
4802
4965
|
required: ["verdicts"]
|
|
4803
4966
|
};
|
|
4804
|
-
var isObj7 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4805
4967
|
var str = (v, max) => typeof v === "string" ? clean(v, max, true) : "";
|
|
4806
4968
|
function pendingListing(v) {
|
|
4807
|
-
if (!
|
|
4969
|
+
if (!isObj3(v) || typeof v.id !== "string")
|
|
4808
4970
|
return null;
|
|
4809
4971
|
const url = safeUrl(v.url);
|
|
4810
4972
|
const title = str(v.title, PENDING_TITLE);
|
|
@@ -4825,7 +4987,7 @@ function pendingListing(v) {
|
|
|
4825
4987
|
}
|
|
4826
4988
|
function judgeState(value) {
|
|
4827
4989
|
const base = wantState(value);
|
|
4828
|
-
const s =
|
|
4990
|
+
const s = isObj3(value) ? value : {};
|
|
4829
4991
|
const pending = (Array.isArray(s.pending) ? s.pending : []).map(pendingListing).filter((l) => l !== null).slice(0, MAX_NEW_PER_RUN);
|
|
4830
4992
|
const failures = typeof s.failures === "number" && s.failures >= 0 ? s.failures : 0;
|
|
4831
4993
|
const retryAt = typeof s.retryAt === "string" && Number.isFinite(Date.parse(s.retryAt)) ? s.retryAt : undefined;
|
|
@@ -4874,13 +5036,14 @@ function closedSource(config) {
|
|
|
4874
5036
|
return config.source === "inbox" || config.source === "shopify" || config.source === "ebay";
|
|
4875
5037
|
}
|
|
4876
5038
|
function limitsText(config) {
|
|
5039
|
+
const currency = currencyCode(config.currency) ?? "";
|
|
4877
5040
|
if (config.maxPrice !== undefined)
|
|
4878
|
-
return `at or under ${money(config.maxPrice,
|
|
4879
|
-
return
|
|
5041
|
+
return `at or under ${money(config.maxPrice, currency)}`;
|
|
5042
|
+
return currency ? `any price, in ${currency}` : "any price";
|
|
4880
5043
|
}
|
|
4881
5044
|
function judgeJob(title, config, pending) {
|
|
4882
5045
|
const closed = closedSource(config);
|
|
4883
|
-
const where = config.source
|
|
5046
|
+
const where = sourceText(config.source).judgeWhere;
|
|
4884
5047
|
const lines = pending.map((l, i) => {
|
|
4885
5048
|
const parts = [
|
|
4886
5049
|
inert(l.title),
|
|
@@ -4926,18 +5089,18 @@ function judgeJob(title, config, pending) {
|
|
|
4926
5089
|
};
|
|
4927
5090
|
}
|
|
4928
5091
|
function parseVerdicts(value, count) {
|
|
4929
|
-
if (!
|
|
5092
|
+
if (!isObj3(value) || !Array.isArray(value.verdicts))
|
|
4930
5093
|
return null;
|
|
4931
5094
|
const out = new Map;
|
|
4932
5095
|
for (const v of value.verdicts) {
|
|
4933
|
-
if (!
|
|
5096
|
+
if (!isObj3(v) || typeof v.n !== "number" || !Number.isInteger(v.n) || v.n < 1 || v.n > count || out.has(v.n))
|
|
4934
5097
|
continue;
|
|
4935
5098
|
if (typeof v.fit !== "string" || !FITS.includes(v.fit))
|
|
4936
5099
|
continue;
|
|
4937
5100
|
out.set(v.n, {
|
|
4938
5101
|
fit: v.fit,
|
|
4939
5102
|
why: inert(str(v.why, WHY_CHARS)),
|
|
4940
|
-
seller: inert(str(v.seller,
|
|
5103
|
+
seller: inert(str(v.seller, SELLER_CHARS2))
|
|
4941
5104
|
});
|
|
4942
5105
|
}
|
|
4943
5106
|
return out;
|
|
@@ -4984,13 +5147,6 @@ function settled(state, told) {
|
|
|
4984
5147
|
failures: 0
|
|
4985
5148
|
};
|
|
4986
5149
|
}
|
|
4987
|
-
var RETRIED3 = new Set([
|
|
4988
|
-
"schema_miss",
|
|
4989
|
-
"malformed",
|
|
4990
|
-
"timeout",
|
|
4991
|
-
"error",
|
|
4992
|
-
"auth_failed"
|
|
4993
|
-
]);
|
|
4994
5150
|
var SAID3 = {
|
|
4995
5151
|
auth_failed: "the model runner could not sign in",
|
|
4996
5152
|
turn_cap: "the check needed more steps than it is allowed",
|
|
@@ -5003,10 +5159,9 @@ var SAID3 = {
|
|
|
5003
5159
|
function judgeFailed(ctx, state, kind, detail) {
|
|
5004
5160
|
const failures = state.failures + 1;
|
|
5005
5161
|
const summary = clean(`judge ${kind} (try ${failures}): ${detail}`, 300, true);
|
|
5006
|
-
|
|
5007
|
-
|
|
5008
|
-
|
|
5009
|
-
return { state: { ...state, failures, retryAt: at }, followUp: wait > 0 ? { at } : {}, summary };
|
|
5162
|
+
const retry = retryOutcome(kind, failures, ctx.now);
|
|
5163
|
+
if (retry) {
|
|
5164
|
+
return { state: { ...state, failures, retryAt: retry.at }, followUp: retry.followUp, summary };
|
|
5010
5165
|
}
|
|
5011
5166
|
const reason = SAID3[kind] ?? "the check failed";
|
|
5012
5167
|
const within = state.pending.filter((l) => withinLimits(l, ctx.config));
|
|
@@ -5092,7 +5247,7 @@ function wantjudgeType(sources, o = {}) {
|
|
|
5092
5247
|
},
|
|
5093
5248
|
async prepare(ctx) {
|
|
5094
5249
|
const state = judgeState(ctx.state);
|
|
5095
|
-
const followUp = ctx.occurrence.dedupeKey.startsWith(
|
|
5250
|
+
const followUp = ctx.occurrence.dedupeKey.startsWith(FOLLOW_UP_PREFIX);
|
|
5096
5251
|
if (state.pending.length > 0) {
|
|
5097
5252
|
const overdue = state.retryAt !== undefined && ctx.now.getTime() > Date.parse(state.retryAt) + RETRY_GRACE_MS;
|
|
5098
5253
|
if (state.failures > 0 && !followUp && !overdue)
|
|
@@ -5244,7 +5399,7 @@ function intakeJob(ctx) {
|
|
|
5244
5399
|
timeoutMs: INTAKE_TIMEOUT_MS
|
|
5245
5400
|
};
|
|
5246
5401
|
}
|
|
5247
|
-
function
|
|
5402
|
+
function isObj5(v) {
|
|
5248
5403
|
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
5249
5404
|
}
|
|
5250
5405
|
function checkDraft(draft, types, tasks = []) {
|
|
@@ -5301,7 +5456,7 @@ function checkDraft(draft, types, tasks = []) {
|
|
|
5301
5456
|
return null;
|
|
5302
5457
|
}
|
|
5303
5458
|
function readDraft(value) {
|
|
5304
|
-
if (!
|
|
5459
|
+
if (!isObj5(value) || !isObj5(value.fields) || typeof value.type !== "string")
|
|
5305
5460
|
return null;
|
|
5306
5461
|
if (value.action !== "create" && value.action !== "edit")
|
|
5307
5462
|
return null;
|
|
@@ -5326,7 +5481,7 @@ function readIntake(result, types, tasks = []) {
|
|
|
5326
5481
|
if (result.kind !== "success")
|
|
5327
5482
|
return { ok: false, retry: false, problem: result.kind };
|
|
5328
5483
|
const v = result.structuredOutput;
|
|
5329
|
-
if (!
|
|
5484
|
+
if (!isObj5(v) || typeof v.reply !== "string" || !INTAKE_STATES.includes(v.state)) {
|
|
5330
5485
|
return {
|
|
5331
5486
|
ok: false,
|
|
5332
5487
|
retry: true,
|
|
@@ -6214,11 +6369,21 @@ class SqliteStore {
|
|
|
6214
6369
|
params.push(expected);
|
|
6215
6370
|
return this.run(`UPDATE occurrences SET ${sets.join(", ")} WHERE seq = ?${guard}`, ...params).changes > 0;
|
|
6216
6371
|
}
|
|
6217
|
-
async deleteOccurrence(id) {
|
|
6372
|
+
async deleteOccurrence(id, guard) {
|
|
6218
6373
|
const seq = seqOf("o", id);
|
|
6219
6374
|
if (seq === null)
|
|
6220
6375
|
return false;
|
|
6221
|
-
|
|
6376
|
+
if (guard === undefined)
|
|
6377
|
+
return this.run("DELETE FROM occurrences WHERE seq = ? AND status = 'queued'", seq).changes > 0;
|
|
6378
|
+
const last = guard.lastEventId === null ? null : seqOf("e", guard.lastEventId);
|
|
6379
|
+
if (guard.lastEventId !== null && last === null)
|
|
6380
|
+
return false;
|
|
6381
|
+
if (last === null) {
|
|
6382
|
+
return this.run("DELETE FROM occurrences WHERE seq = ? AND status = 'queued' AND record IS NULL AND NOT EXISTS (SELECT 1 FROM events WHERE occurrence_id = ?)", seq, id).changes > 0;
|
|
6383
|
+
}
|
|
6384
|
+
return this.run(`DELETE FROM occurrences WHERE seq = ? AND status = 'queued' AND record IS NULL
|
|
6385
|
+
AND EXISTS (SELECT 1 FROM events WHERE seq = ? AND occurrence_id = ?)
|
|
6386
|
+
AND NOT EXISTS (SELECT 1 FROM events WHERE occurrence_id = ? AND seq > ?)`, seq, last, id, id, last).changes > 0;
|
|
6222
6387
|
}
|
|
6223
6388
|
async requeueRunning(lane) {
|
|
6224
6389
|
const rows = lane === undefined ? this.all("SELECT seq FROM occurrences WHERE status = 'running' ORDER BY seq") : this.all("SELECT seq FROM occurrences WHERE status = 'running' AND lane = ? ORDER BY seq", lane);
|
|
@@ -6291,7 +6456,7 @@ class SqliteStore {
|
|
|
6291
6456
|
}
|
|
6292
6457
|
async listSeries(taskId, filter = {}) {
|
|
6293
6458
|
const points = filter.since === undefined ? this.all("SELECT * FROM series WHERE task_id = ? ORDER BY at, seq", taskId) : this.all("SELECT * FROM series WHERE task_id = ? AND at >= ? ORDER BY at, seq", taskId, filter.since);
|
|
6294
|
-
const kept = filter.limit === undefined ? points : points.slice(-filter.limit);
|
|
6459
|
+
const kept = filter.limit === undefined ? points : !(filter.limit >= 1) ? [] : points.slice(-filter.limit);
|
|
6295
6460
|
return kept.map(toPoint);
|
|
6296
6461
|
}
|
|
6297
6462
|
async addFinding(f) {
|
|
@@ -6366,10 +6531,10 @@ class SqliteStore {
|
|
|
6366
6531
|
WHERE occurrence_id = ? AND user_id = ? AND retry_at IS NOT NULL`, at, occurrenceId, userId);
|
|
6367
6532
|
return changes === 1 ? this.delivery(occurrenceId, userId) : null;
|
|
6368
6533
|
}
|
|
6369
|
-
async settleDelivery(occurrenceId, userId, settle) {
|
|
6534
|
+
async settleDelivery(occurrenceId, userId, settle, expected) {
|
|
6370
6535
|
const { changes } = this.run(`UPDATE deliveries SET status = ?, message_id = ?, error = ?, attempts = ?, deferrals = ?, retry_at = ?, settled_at = ?
|
|
6371
|
-
WHERE occurrence_id = ? AND user_id =
|
|
6372
|
-
if (changes === 0)
|
|
6536
|
+
WHERE occurrence_id = ? AND user_id = ?${expected === undefined ? "" : " AND status = ?"}`, settle.status, settle.messageId ?? null, settle.error ?? null, settle.attempts, settle.deferrals, settle.retryAt, settle.at, occurrenceId, userId, ...expected === undefined ? [] : [expected]);
|
|
6537
|
+
if (changes === 0 && !this.delivery(occurrenceId, userId))
|
|
6373
6538
|
throw new Error(`no delivery ${occurrenceId}:${userId}`);
|
|
6374
6539
|
return this.delivery(occurrenceId, userId);
|
|
6375
6540
|
}
|
|
@@ -7233,7 +7398,7 @@ async function rescheduleOwned(d, owner, zoneChanged, before) {
|
|
|
7233
7398
|
return moved;
|
|
7234
7399
|
}
|
|
7235
7400
|
async function rescheduleKeepingSnoozes(d, task, owner, schedule, actorId) {
|
|
7236
|
-
const { task: updated, jobOut } = await reschedule(d.store, task, owner, schedule, actorId, d.clock.now());
|
|
7401
|
+
const { task: updated, jobOut } = await reschedule(d.store, task, owner, schedule, actorId, d.clock.now(), d.types[task.type]);
|
|
7237
7402
|
return { task: updated, jobOut };
|
|
7238
7403
|
}
|
|
7239
7404
|
var JOB_OUT_NOTE = "A run of this task is already with the model runner: it still counts, and the new schedule follows it.";
|
|
@@ -7437,8 +7602,7 @@ async function deleteLocked(d, user, taskId) {
|
|
|
7437
7602
|
return { ok: false, error: NO_SUCH_TASK };
|
|
7438
7603
|
const at = d.clock.now().toISOString();
|
|
7439
7604
|
for (const o of await d.store.listOccurrences({ taskId: task.id, status: "queued" })) {
|
|
7440
|
-
|
|
7441
|
-
await d.store.deleteOccurrence(o.id);
|
|
7605
|
+
await deleteIdleRun(d.store, o);
|
|
7442
7606
|
}
|
|
7443
7607
|
await d.store.updateTask(task.id, { status: "archived", at });
|
|
7444
7608
|
await d.store.addTaskEvent({ taskId: task.id, actorId: user.id, kind: "archived", detail: "deleted by the owner", at });
|
|
@@ -7530,6 +7694,10 @@ function nearPattern(near) {
|
|
|
7530
7694
|
const literal = near.trim().replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
7531
7695
|
return `${literal}[^0-9]{0,40}([0-9](?:[0-9.,]{0,14}[0-9])?)`;
|
|
7532
7696
|
}
|
|
7697
|
+
function patternRefusal(pattern) {
|
|
7698
|
+
const problem = pattern === undefined ? null : patternProblem(pattern);
|
|
7699
|
+
return problem ? `That \`near\` text cannot be used: ${problem}` : null;
|
|
7700
|
+
}
|
|
7533
7701
|
function priceSettingsPlan(input) {
|
|
7534
7702
|
const hours = input.hours ?? DEFAULT_POLL_HOURS;
|
|
7535
7703
|
if (!Number.isInteger(hours) || hours < 1 || hours > MAX_POLL_HOURS)
|
|
@@ -7600,7 +7768,11 @@ async function startPrice(d, user, input) {
|
|
|
7600
7768
|
if (capped)
|
|
7601
7769
|
return { ok: false, error: capped };
|
|
7602
7770
|
const { name, hours, drop, baseline } = settings;
|
|
7603
|
-
const
|
|
7771
|
+
const pattern = near ? nearPattern(near) : undefined;
|
|
7772
|
+
const refused = patternRefusal(pattern);
|
|
7773
|
+
if (refused)
|
|
7774
|
+
return { ok: false, error: refused };
|
|
7775
|
+
const config = { url, dropPercent: drop, baseline, ...pattern !== undefined ? { pattern } : {} };
|
|
7604
7776
|
return { ok: true, pending: { url, title: name || titleFor(url), hours, config } };
|
|
7605
7777
|
}
|
|
7606
7778
|
async function trackPrice(d, user, input) {
|
|
@@ -10104,6 +10276,8 @@ async function pressLocked(d, user, ref) {
|
|
|
10104
10276
|
} catch (err) {
|
|
10105
10277
|
if (err instanceof ReplyRefusedError)
|
|
10106
10278
|
return { ok: false, error: err.message };
|
|
10279
|
+
if (err instanceof ConsentError)
|
|
10280
|
+
return { ok: false, error: "That is not yours to answer." };
|
|
10107
10281
|
throw err;
|
|
10108
10282
|
}
|
|
10109
10283
|
}
|
|
@@ -14930,7 +15104,7 @@ async function sourcesReadyAnswer(w, user, id) {
|
|
|
14930
15104
|
if (followUp)
|
|
14931
15105
|
return fullerPass(w, user, task, owner, followUp, held, now);
|
|
14932
15106
|
try {
|
|
14933
|
-
await reschedule(w.d.store, task, owner, { kind: "once", at: now.toISOString() }, user.id, now);
|
|
15107
|
+
await reschedule(w.d.store, task, owner, { kind: "once", at: now.toISOString() }, user.id, now, w.d.types[task.type]);
|
|
14934
15108
|
} catch (err) {
|
|
14935
15109
|
if (err instanceof ScheduleError)
|
|
14936
15110
|
return problem(409, "conflict", "That research request has started.");
|
|
@@ -14946,7 +15120,7 @@ async function fullerPass(w, user, task, owner, followUp, held, now) {
|
|
|
14946
15120
|
const store = w.d.store;
|
|
14947
15121
|
let next;
|
|
14948
15122
|
try {
|
|
14949
|
-
next = (await reschedule(store, task, owner, { kind: "once", at: now.toISOString() }, user.id, now)).next;
|
|
15123
|
+
next = (await reschedule(store, task, owner, { kind: "once", at: now.toISOString() }, user.id, now, w.d.types[task.type])).next;
|
|
14950
15124
|
} catch (err) {
|
|
14951
15125
|
if (err instanceof ScheduleError)
|
|
14952
15126
|
return problem(409, "conflict", "That research request has started.");
|
|
@@ -14954,7 +15128,7 @@ async function fullerPass(w, user, task, owner, followUp, held, now) {
|
|
|
14954
15128
|
}
|
|
14955
15129
|
if (!next)
|
|
14956
15130
|
return problem(409, "conflict", "That research request could not be started just now; try again later.");
|
|
14957
|
-
if (!await store
|
|
15131
|
+
if (!await deleteIdleRun(store, followUp)) {
|
|
14958
15132
|
await store.deleteOccurrence(next.id);
|
|
14959
15133
|
return problem(409, "conflict", "That research request has started.");
|
|
14960
15134
|
}
|