@rackbops/plugin-tracker 0.35.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -7
- package/dist/plugin.js +1394 -623
- package/package.json +10 -3
package/dist/plugin.js
CHANGED
|
@@ -192,15 +192,15 @@ function reads(instant, zone, d, h, min) {
|
|
|
192
192
|
}
|
|
193
193
|
function zonedInstant(year, month, day, hour, minute, zone) {
|
|
194
194
|
const wanted = Date.UTC(year, month - 1, day, hour, minute);
|
|
195
|
-
const
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
const
|
|
200
|
-
const
|
|
201
|
-
if (
|
|
202
|
-
return new Date(
|
|
203
|
-
return new Date(Math.max(
|
|
195
|
+
const offsets = new Set;
|
|
196
|
+
for (const at of [wanted - 86400000, wanted, wanted + 86400000]) {
|
|
197
|
+
offsets.add(zoneOffsetMinutes(new Date(at), zone));
|
|
198
|
+
}
|
|
199
|
+
const candidates = [...offsets].map((offset) => wanted - offset * 60000);
|
|
200
|
+
const matching = candidates.filter((c) => reads(c, zone, day, hour, minute));
|
|
201
|
+
if (matching.length > 0)
|
|
202
|
+
return new Date(Math.min(...matching));
|
|
203
|
+
return new Date(Math.max(...candidates));
|
|
204
204
|
}
|
|
205
205
|
|
|
206
206
|
// node_modules/@rackbops/docket-core/dist/budget.js
|
|
@@ -223,16 +223,20 @@ function budgetDay(now) {
|
|
|
223
223
|
end: end.toISOString()
|
|
224
224
|
};
|
|
225
225
|
}
|
|
226
|
+
function costOf(usd) {
|
|
227
|
+
return typeof usd === "number" && Number.isFinite(usd) && usd > 0 ? usd : 0;
|
|
228
|
+
}
|
|
226
229
|
function spent(rows) {
|
|
227
|
-
return rows.reduce((s, r) => ({ usd: s.usd + r.costUsd, calls: s.calls + r.calls }), {
|
|
230
|
+
return rows.reduce((s, r) => ({ usd: s.usd + costOf(r.costUsd), calls: s.calls + r.calls }), {
|
|
228
231
|
usd: 0,
|
|
229
232
|
calls: 0
|
|
230
233
|
});
|
|
231
234
|
}
|
|
235
|
+
var USD_EPSILON = 0.000000001;
|
|
232
236
|
function reached(used, limits) {
|
|
233
237
|
if (limits.calls !== null && used.calls >= limits.calls)
|
|
234
238
|
return "calls";
|
|
235
|
-
if (limits.usd !== null && used.usd >= limits.usd)
|
|
239
|
+
if (limits.usd !== null && used.usd >= limits.usd - USD_EPSILON)
|
|
236
240
|
return "usd";
|
|
237
241
|
return null;
|
|
238
242
|
}
|
|
@@ -261,7 +265,7 @@ async function charge(store, c) {
|
|
|
261
265
|
source: c.source,
|
|
262
266
|
key: c.key ?? null,
|
|
263
267
|
calls: c.calls,
|
|
264
|
-
costUsd: c.costUsd
|
|
268
|
+
costUsd: costOf(c.costUsd),
|
|
265
269
|
at: c.at.toISOString()
|
|
266
270
|
});
|
|
267
271
|
}
|
|
@@ -426,6 +430,10 @@ async function respondToInvite(store, task, recipientId, answer, now, occurrence
|
|
|
426
430
|
return { recipient, reply, block };
|
|
427
431
|
}
|
|
428
432
|
async function optOut(store, task, recipientId, now, occurrenceId = null) {
|
|
433
|
+
const existing = (await store.listRecipients(task.id)).find((r) => r.userId === recipientId);
|
|
434
|
+
if (existing?.state !== "accepted") {
|
|
435
|
+
throw new ConsentError(`${recipientId} is not an accepted recipient of task ${task.id}`);
|
|
436
|
+
}
|
|
429
437
|
const at = now.toISOString();
|
|
430
438
|
await store.setRecipient(task.id, recipientId, "opted_out", at);
|
|
431
439
|
await store.addTaskEvent({
|
|
@@ -459,6 +467,22 @@ function isLane(value) {
|
|
|
459
467
|
return LANES.includes(value);
|
|
460
468
|
}
|
|
461
469
|
|
|
470
|
+
// node_modules/@rackbops/docket-core/dist/local-date.js
|
|
471
|
+
var DAY_MS = 86400000;
|
|
472
|
+
function daysInMonth(year, month) {
|
|
473
|
+
return new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
474
|
+
}
|
|
475
|
+
function dayNumber(d) {
|
|
476
|
+
return Math.floor(Date.UTC(d.year, d.month - 1, d.day) / DAY_MS);
|
|
477
|
+
}
|
|
478
|
+
function fromDayNumber(n) {
|
|
479
|
+
const dt = new Date(n * DAY_MS);
|
|
480
|
+
return { year: dt.getUTCFullYear(), month: dt.getUTCMonth() + 1, day: dt.getUTCDate() };
|
|
481
|
+
}
|
|
482
|
+
function addDays(d, days) {
|
|
483
|
+
return fromDayNumber(dayNumber(d) + days);
|
|
484
|
+
}
|
|
485
|
+
|
|
462
486
|
// node_modules/@rackbops/docket-core/dist/schedule.js
|
|
463
487
|
var SCHEDULE_KINDS = ["once", "calendar", "poll", "period"];
|
|
464
488
|
function isScheduleKind(value) {
|
|
@@ -466,8 +490,12 @@ function isScheduleKind(value) {
|
|
|
466
490
|
}
|
|
467
491
|
var MIN_POLL_MINUTES = 5;
|
|
468
492
|
var MINUTE_MS = 60000;
|
|
469
|
-
var DAY_MS = 86400000;
|
|
470
493
|
var HORIZON_DAYS = 366 * 3;
|
|
494
|
+
var MAX_CALENDAR_EVERY = {
|
|
495
|
+
day: HORIZON_DAYS - 1,
|
|
496
|
+
week: Math.floor((HORIZON_DAYS - 1) / 7),
|
|
497
|
+
month: 36
|
|
498
|
+
};
|
|
471
499
|
var HORIZON_PERIODS = 1e5;
|
|
472
500
|
function parseLocalDate(value) {
|
|
473
501
|
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
|
|
@@ -485,16 +513,6 @@ function formatLocalDate(d) {
|
|
|
485
513
|
const dd = String(d.day).padStart(2, "0");
|
|
486
514
|
return `${d.year}-${mm}-${dd}`;
|
|
487
515
|
}
|
|
488
|
-
function daysInMonth(year, month) {
|
|
489
|
-
return new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
490
|
-
}
|
|
491
|
-
function dayNumber(d) {
|
|
492
|
-
return Math.floor(Date.UTC(d.year, d.month - 1, d.day) / DAY_MS);
|
|
493
|
-
}
|
|
494
|
-
function fromDayNumber(n) {
|
|
495
|
-
const dt = new Date(n * DAY_MS);
|
|
496
|
-
return { year: dt.getUTCFullYear(), month: dt.getUTCMonth() + 1, day: dt.getUTCDate() };
|
|
497
|
-
}
|
|
498
516
|
function weekdayOf(d) {
|
|
499
517
|
return new Date(Date.UTC(d.year, d.month - 1, d.day)).getUTCDay();
|
|
500
518
|
}
|
|
@@ -536,8 +554,14 @@ function calendarProblems(schedule) {
|
|
|
536
554
|
if (!Number.isInteger(schedule.every) || schedule.every < 1) {
|
|
537
555
|
problems.push("calendar.every must be a whole number >= 1");
|
|
538
556
|
}
|
|
539
|
-
if (!["day", "week", "month"].includes(schedule.unit))
|
|
557
|
+
if (!["day", "week", "month"].includes(schedule.unit)) {
|
|
540
558
|
problems.push("calendar.unit is unknown");
|
|
559
|
+
} else if (Number.isInteger(schedule.every) && schedule.every >= 1) {
|
|
560
|
+
const max = MAX_CALENDAR_EVERY[schedule.unit];
|
|
561
|
+
if (schedule.every > max) {
|
|
562
|
+
problems.push(`calendar.every must be at most ${max} for unit ${schedule.unit}`);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
541
565
|
if (!parseLocalDate(schedule.start))
|
|
542
566
|
problems.push("calendar.start must be a YYYY-MM-DD date");
|
|
543
567
|
if (schedule.weekdays !== undefined) {
|
|
@@ -659,11 +683,23 @@ function nextPollDue(schedule, after) {
|
|
|
659
683
|
return null;
|
|
660
684
|
return new Date(due);
|
|
661
685
|
}
|
|
686
|
+
var LONGEST_PERIOD_DAYS = { day: 1, week: 7, month: 31, year: 366 };
|
|
687
|
+
function firstPeriod(schedule, after, zone) {
|
|
688
|
+
const anchor = parseLocalDate(schedule.anchor);
|
|
689
|
+
if (!anchor)
|
|
690
|
+
return 0;
|
|
691
|
+
const local = wallClock(after, zone);
|
|
692
|
+
const elapsed = dayNumber({ year: local.year, month: local.month, day: local.day }) - dayNumber(anchor);
|
|
693
|
+
const longest = LONGEST_PERIOD_DAYS[schedule.unit] * schedule.every;
|
|
694
|
+
if (!(longest >= 1) || (schedule.leadDays ?? 0) < 0)
|
|
695
|
+
return 0;
|
|
696
|
+
return Math.max(0, Math.floor(elapsed / longest) - 1);
|
|
697
|
+
}
|
|
662
698
|
function nextPeriodDue(schedule, after, ctx) {
|
|
663
699
|
const hour = schedule.hour ?? ctx.preferredHour;
|
|
664
700
|
const minute = schedule.minute ?? 0;
|
|
665
701
|
const lead = schedule.leadDays ?? 0;
|
|
666
|
-
for (let n =
|
|
702
|
+
for (let n = firstPeriod(schedule, after, ctx.zone);n < HORIZON_PERIODS; n++) {
|
|
667
703
|
const date = periodDateAt(schedule, n);
|
|
668
704
|
if (!date)
|
|
669
705
|
return null;
|
|
@@ -741,8 +777,15 @@ function followUpKey(occurrenceId) {
|
|
|
741
777
|
return `${FOLLOW_UP_PREFIX}${occurrenceId}`;
|
|
742
778
|
}
|
|
743
779
|
var FOLLOW_UP_PREFIX = "followup:";
|
|
780
|
+
function runNowKey(taskId, requestId) {
|
|
781
|
+
if (!REQUEST_ID.test(requestId))
|
|
782
|
+
throw new Error("runNowKey: requestId must be 1 to 64 of A-Z a-z 0-9 - _");
|
|
783
|
+
return `${RUN_NOW_PREFIX}${taskId}:${requestId}`;
|
|
784
|
+
}
|
|
785
|
+
var REQUEST_ID = /^[A-Za-z0-9_-]{1,64}$/;
|
|
786
|
+
var RUN_NOW_PREFIX = "now:";
|
|
744
787
|
function isOffSchedule(dedupeKey) {
|
|
745
|
-
return dedupeKey.startsWith(SNOOZE_PREFIX) || dedupeKey.startsWith(FOLLOW_UP_PREFIX);
|
|
788
|
+
return dedupeKey.startsWith(SNOOZE_PREFIX) || dedupeKey.startsWith(FOLLOW_UP_PREFIX) || dedupeKey.startsWith(RUN_NOW_PREFIX);
|
|
746
789
|
}
|
|
747
790
|
|
|
748
791
|
// node_modules/@rackbops/docket-core/dist/ports.js
|
|
@@ -805,8 +848,8 @@ async function deliver(store, notifier, occurrence, ownerId, targets, message, o
|
|
|
805
848
|
}
|
|
806
849
|
const target = targets.get(row.userId);
|
|
807
850
|
if (!target) {
|
|
808
|
-
await finalFail(store, row, "no longer a recipient", now());
|
|
809
|
-
report.failed.push(row.userId);
|
|
851
|
+
const ended = await finalFail(store, row, "no longer a recipient", now());
|
|
852
|
+
(ended ? report.failed : report.untried).push(row.userId);
|
|
810
853
|
continue;
|
|
811
854
|
}
|
|
812
855
|
const claim = await store.claimDelivery(occurrence.id, row.userId, now().toISOString());
|
|
@@ -861,18 +904,20 @@ async function sendOne(store, notifier, occurrence, claim, target, ownerId, mess
|
|
|
861
904
|
report.sent.push(claim.userId);
|
|
862
905
|
}
|
|
863
906
|
async function finalFail(store, row, error, at) {
|
|
864
|
-
await store.settleDelivery(row.occurrenceId, row.userId, {
|
|
907
|
+
const after = await store.settleDelivery(row.occurrenceId, row.userId, {
|
|
865
908
|
status: "failed",
|
|
866
909
|
error,
|
|
867
910
|
attempts: row.attempts,
|
|
868
911
|
deferrals: row.deferrals,
|
|
869
912
|
retryAt: null,
|
|
870
913
|
at: at.toISOString()
|
|
871
|
-
});
|
|
914
|
+
}, row.status);
|
|
915
|
+
return after.status === "failed" && after.settledAt === at.toISOString();
|
|
872
916
|
}
|
|
873
917
|
async function dropOwed(store, rows, error, at) {
|
|
874
918
|
for (const row of rows)
|
|
875
|
-
|
|
919
|
+
if (row.retryAt !== null)
|
|
920
|
+
await finalFail(store, row, error, at);
|
|
876
921
|
}
|
|
877
922
|
async function settleStaleClaims(store, now, all = false) {
|
|
878
923
|
const cutoff = new Date(now.getTime() - STALE_CLAIM_MS).toISOString();
|
|
@@ -880,15 +925,16 @@ async function settleStaleClaims(store, now, all = false) {
|
|
|
880
925
|
for (const row of await store.listDeliveries({ status: "claimed" })) {
|
|
881
926
|
if (!all && (row.claimedAt ?? "") > cutoff)
|
|
882
927
|
continue;
|
|
883
|
-
await store.settleDelivery(row.occurrenceId, row.userId, {
|
|
928
|
+
const after = await store.settleDelivery(row.occurrenceId, row.userId, {
|
|
884
929
|
status: "unconfirmed",
|
|
885
930
|
error: "claimed, never settled",
|
|
886
931
|
attempts: row.attempts,
|
|
887
932
|
deferrals: row.deferrals,
|
|
888
933
|
retryAt: null,
|
|
889
934
|
at: now.toISOString()
|
|
890
|
-
});
|
|
891
|
-
|
|
935
|
+
}, "claimed");
|
|
936
|
+
if (after.status === "unconfirmed" && after.settledAt === now.toISOString())
|
|
937
|
+
settled += 1;
|
|
892
938
|
}
|
|
893
939
|
return settled;
|
|
894
940
|
}
|
|
@@ -904,7 +950,7 @@ async function freshRoundForPause(store, rows) {
|
|
|
904
950
|
deferrals: 0,
|
|
905
951
|
retryAt: row.retryAt,
|
|
906
952
|
at: row.settledAt ?? row.createdAt
|
|
907
|
-
});
|
|
953
|
+
}, row.status);
|
|
908
954
|
}
|
|
909
955
|
}
|
|
910
956
|
|
|
@@ -1002,6 +1048,14 @@ var LATE_AFTER_MS = 5 * 60000;
|
|
|
1002
1048
|
class ScheduleError extends Error {
|
|
1003
1049
|
name = "ScheduleError";
|
|
1004
1050
|
}
|
|
1051
|
+
function checkSchedule(schedule, type) {
|
|
1052
|
+
if (type && !type.schedule.includes(schedule.kind)) {
|
|
1053
|
+
throw new ScheduleError(`${type.id} does not accept a ${schedule.kind} schedule`);
|
|
1054
|
+
}
|
|
1055
|
+
const problems = scheduleProblems(schedule);
|
|
1056
|
+
if (problems.length > 0)
|
|
1057
|
+
throw new ScheduleError(problems.join("; "));
|
|
1058
|
+
}
|
|
1005
1059
|
function isLate(occurrence, now) {
|
|
1006
1060
|
return now.getTime() - Date.parse(occurrence.dueAt) > LATE_AFTER_MS;
|
|
1007
1061
|
}
|
|
@@ -1037,13 +1091,19 @@ function firstDueAfter(schedule, fresh, now) {
|
|
|
1037
1091
|
async function cancelScheduledRuns(store, taskId) {
|
|
1038
1092
|
let removed = 0;
|
|
1039
1093
|
for (const o of await store.listOccurrences({ taskId, status: "queued" })) {
|
|
1040
|
-
if (isOffSchedule(o.dedupeKey)
|
|
1094
|
+
if (isOffSchedule(o.dedupeKey))
|
|
1041
1095
|
continue;
|
|
1042
|
-
if (await store
|
|
1096
|
+
if (await deleteIdleRun(store, o))
|
|
1043
1097
|
removed += 1;
|
|
1044
1098
|
}
|
|
1045
1099
|
return removed;
|
|
1046
1100
|
}
|
|
1101
|
+
async function deleteIdleRun(store, o) {
|
|
1102
|
+
const lastEventId = (await store.listEvents(o.id)).at(-1)?.id ?? null;
|
|
1103
|
+
if (hasFired(o) || await hasJobOut(store, o))
|
|
1104
|
+
return false;
|
|
1105
|
+
return store.deleteOccurrence(o.id, { lastEventId });
|
|
1106
|
+
}
|
|
1047
1107
|
async function anyJobOut(store, taskId) {
|
|
1048
1108
|
for (const o of await store.listOccurrences({ taskId, status: "queued" })) {
|
|
1049
1109
|
if (!hasFired(o) && await hasJobOut(store, o))
|
|
@@ -1051,12 +1111,9 @@ async function anyJobOut(store, taskId) {
|
|
|
1051
1111
|
}
|
|
1052
1112
|
return false;
|
|
1053
1113
|
}
|
|
1054
|
-
async function reschedule(store, task, owner, schedule, actorId, now) {
|
|
1055
|
-
if (schedule)
|
|
1056
|
-
|
|
1057
|
-
if (problems.length > 0)
|
|
1058
|
-
throw new ScheduleError(problems.join("; "));
|
|
1059
|
-
}
|
|
1114
|
+
async function reschedule(store, task, owner, schedule, actorId, now, type) {
|
|
1115
|
+
if (schedule)
|
|
1116
|
+
checkSchedule(schedule, type);
|
|
1060
1117
|
const jobOut = await anyJobOut(store, task.id);
|
|
1061
1118
|
if (jobOut && (schedule?.kind === "once" || task.schedule?.kind === "once")) {
|
|
1062
1119
|
throw new ScheduleError("its run is with the runner now; change when it runs once that run is back");
|
|
@@ -1094,7 +1151,11 @@ function clean(text, max, oneLine = false) {
|
|
|
1094
1151
|
if (oneLine)
|
|
1095
1152
|
out = out.replace(/\s+/g, " ");
|
|
1096
1153
|
out = out.trim();
|
|
1097
|
-
|
|
1154
|
+
if (out.length <= max)
|
|
1155
|
+
return out;
|
|
1156
|
+
if (max < 4)
|
|
1157
|
+
return out.slice(0, Math.max(0, max));
|
|
1158
|
+
return `${out.slice(0, max - 3).trimEnd()}...`;
|
|
1098
1159
|
}
|
|
1099
1160
|
|
|
1100
1161
|
// node_modules/@rackbops/docket-core/dist/dispatch.js
|
|
@@ -1223,10 +1284,14 @@ class Lanes {
|
|
|
1223
1284
|
if ("error" in loaded || !loaded.type.onReply)
|
|
1224
1285
|
return null;
|
|
1225
1286
|
const occurrence = input.occurrenceId ? await store.getOccurrence(input.occurrenceId) : null;
|
|
1226
|
-
if (!occurrence)
|
|
1287
|
+
if (!occurrence || occurrence.taskId !== task.id)
|
|
1227
1288
|
return null;
|
|
1228
1289
|
const ctx = await this.context(loaded, occurrence, now);
|
|
1229
|
-
const outcome = await loaded.type.onReply({ ...ctx, reply });
|
|
1290
|
+
const outcome = { ...await loaded.type.onReply({ ...ctx, reply }) };
|
|
1291
|
+
if (input.userId !== task.ownerId) {
|
|
1292
|
+
delete outcome.complete;
|
|
1293
|
+
delete outcome.snoozeUntil;
|
|
1294
|
+
}
|
|
1230
1295
|
await this.apply(task, outcome, now, null, `reply:${reply.id}`);
|
|
1231
1296
|
if (outcome.complete)
|
|
1232
1297
|
await this.complete(task, input.userId, now);
|
|
@@ -1375,8 +1440,7 @@ class Lanes {
|
|
|
1375
1440
|
const { store } = this.d;
|
|
1376
1441
|
const at = now.toISOString();
|
|
1377
1442
|
for (const o of await store.listOccurrences({ taskId: task.id, status: "queued" })) {
|
|
1378
|
-
|
|
1379
|
-
await store.deleteOccurrence(o.id);
|
|
1443
|
+
await deleteIdleRun(store, o);
|
|
1380
1444
|
}
|
|
1381
1445
|
await store.updateTask(task.id, { status: "done", at });
|
|
1382
1446
|
await store.addTaskEvent({ taskId: task.id, actorId, kind: "completed", detail: "", at });
|
|
@@ -1996,14 +2060,8 @@ async function createTask(store, actor, owner, input, now) {
|
|
|
1996
2060
|
if (!actor.admin && actor.userId !== owner.id) {
|
|
1997
2061
|
throw new TaskError("only the owner or an admin creates a task for a person");
|
|
1998
2062
|
}
|
|
1999
|
-
if (input.schedule)
|
|
2000
|
-
|
|
2001
|
-
throw new ScheduleError(`${input.type.id} does not accept a ${input.schedule.kind} schedule`);
|
|
2002
|
-
}
|
|
2003
|
-
const problems = scheduleProblems(input.schedule);
|
|
2004
|
-
if (problems.length > 0)
|
|
2005
|
-
throw new ScheduleError(problems.join("; "));
|
|
2006
|
-
}
|
|
2063
|
+
if (input.schedule)
|
|
2064
|
+
checkSchedule(input.schedule, input.type);
|
|
2007
2065
|
const at = now.toISOString();
|
|
2008
2066
|
const task = await store.createTask({
|
|
2009
2067
|
ownerId: owner.id,
|
|
@@ -2053,27 +2111,12 @@ var DAY_TIME_RE = new RegExp(`^(?:on )?(${DAY})(?: (?:at )?(${TIME}))?$`);
|
|
|
2053
2111
|
var TIME_DAY_RE = new RegExp(`^(?:at )?(${TIME})(?: (?:on )?(${DAY}))?$`);
|
|
2054
2112
|
var MAX_AHEAD_YEARS = 5;
|
|
2055
2113
|
var MAX_AHEAD_MS = MAX_AHEAD_YEARS * 365.25 * 86400000;
|
|
2056
|
-
function knownZone(zone) {
|
|
2057
|
-
try {
|
|
2058
|
-
new Intl.DateTimeFormat("en-US", { timeZone: zone });
|
|
2059
|
-
return true;
|
|
2060
|
-
} catch {
|
|
2061
|
-
return false;
|
|
2062
|
-
}
|
|
2063
|
-
}
|
|
2064
2114
|
var HINT = `Try "in 2 hours", "9am", "tomorrow 9am", "friday 17:30" or "2026-10-01 9:00".`;
|
|
2065
2115
|
function fail(text, why) {
|
|
2066
2116
|
return { ok: false, error: `${why ?? `I couldn't read "${text}" as a time.`} ${HINT}` };
|
|
2067
2117
|
}
|
|
2068
|
-
function daysIn(year, month) {
|
|
2069
|
-
return new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
2070
|
-
}
|
|
2071
|
-
function addDays(date, days) {
|
|
2072
|
-
const d = new Date(Date.UTC(date.year, date.month - 1, date.day + days));
|
|
2073
|
-
return { year: d.getUTCFullYear(), month: d.getUTCMonth() + 1, day: d.getUTCDate() };
|
|
2074
|
-
}
|
|
2075
2118
|
function validDate(date) {
|
|
2076
|
-
return date.month >= 1 && date.month <= 12 && date.day >= 1 && date.day <=
|
|
2119
|
+
return date.month >= 1 && date.month <= 12 && date.day >= 1 && date.day <= daysInMonth(date.year, date.month);
|
|
2077
2120
|
}
|
|
2078
2121
|
function parseTime(word) {
|
|
2079
2122
|
if (word === "noon")
|
|
@@ -2132,7 +2175,7 @@ function resolve(text, now, options) {
|
|
|
2132
2175
|
return fail(text, `${text} is more than ${MAX_AHEAD_YEARS} years away.`);
|
|
2133
2176
|
return { ok: true, at: new Date(now.getTime() + n * unit) };
|
|
2134
2177
|
}
|
|
2135
|
-
if (!
|
|
2178
|
+
if (!isTimeZone(options.zone))
|
|
2136
2179
|
return fail(text, `I don't know the time zone ${options.zone}.`);
|
|
2137
2180
|
let dayWord;
|
|
2138
2181
|
let timeWord;
|
|
@@ -2283,7 +2326,7 @@ function priceFromJson(value, depth = 0) {
|
|
|
2283
2326
|
}
|
|
2284
2327
|
return null;
|
|
2285
2328
|
}
|
|
2286
|
-
var JSON_LD = /<script[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
|
|
2329
|
+
var JSON_LD = /<script[^>]*type\s*=\s*(?:["']application\/ld\+json["']|application\/ld\+json(?=[\s>]))[^>]*>([\s\S]*?)<\/script>/gi;
|
|
2287
2330
|
function jsonLdBlocks(html) {
|
|
2288
2331
|
const blocks = [];
|
|
2289
2332
|
for (const m of html.matchAll(JSON_LD)) {
|
|
@@ -2318,14 +2361,85 @@ function priceFromMeta(html) {
|
|
|
2318
2361
|
}
|
|
2319
2362
|
return null;
|
|
2320
2363
|
}
|
|
2321
|
-
|
|
2322
|
-
|
|
2364
|
+
var MAX_PATTERN_CHARS = 300;
|
|
2365
|
+
var MAX_PATTERN_BODY_CHARS = 500000;
|
|
2366
|
+
function patternProblem(pattern) {
|
|
2367
|
+
if (pattern.length > MAX_PATTERN_CHARS) {
|
|
2368
|
+
return `the pattern is longer than ${MAX_PATTERN_CHARS} characters`;
|
|
2369
|
+
}
|
|
2323
2370
|
try {
|
|
2324
|
-
|
|
2371
|
+
new RegExp(pattern, "i");
|
|
2325
2372
|
} catch {
|
|
2326
|
-
return
|
|
2373
|
+
return "the pattern is not a valid regular expression";
|
|
2374
|
+
}
|
|
2375
|
+
return repeatsAnAmbiguousGroup(pattern) ? "the pattern repeats a group that itself repeats or has a |, like (a+)+ or (a|b)*, " + "which can stall the tracker; use a character class, like [\\d,]+" : null;
|
|
2376
|
+
}
|
|
2377
|
+
function unboundedAt(p, i) {
|
|
2378
|
+
const c = p[i];
|
|
2379
|
+
return c === "*" || c === "+" || /^\{\d+,\}/.test(p.slice(i));
|
|
2380
|
+
}
|
|
2381
|
+
function repeatsAt(p, i) {
|
|
2382
|
+
if (unboundedAt(p, i))
|
|
2383
|
+
return true;
|
|
2384
|
+
const m = /^\{(\d+)(?:,(\d+))?\}/.exec(p.slice(i));
|
|
2385
|
+
return m !== null && Number(m[2] ?? m[1]) > 1;
|
|
2386
|
+
}
|
|
2387
|
+
function repeatsAnAmbiguousGroup(p) {
|
|
2388
|
+
const open = [];
|
|
2389
|
+
const top = () => open[open.length - 1];
|
|
2390
|
+
let inside = false;
|
|
2391
|
+
for (let i = 0;i < p.length; i++) {
|
|
2392
|
+
const c = p[i];
|
|
2393
|
+
if (c === "\\") {
|
|
2394
|
+
i++;
|
|
2395
|
+
continue;
|
|
2396
|
+
}
|
|
2397
|
+
if (inside) {
|
|
2398
|
+
if (c === "]")
|
|
2399
|
+
inside = false;
|
|
2400
|
+
continue;
|
|
2401
|
+
}
|
|
2402
|
+
if (c === "[") {
|
|
2403
|
+
inside = true;
|
|
2404
|
+
if (p[i + 1] === "]")
|
|
2405
|
+
i++;
|
|
2406
|
+
continue;
|
|
2407
|
+
}
|
|
2408
|
+
if (c === "(") {
|
|
2409
|
+
open.push({ repeats: false, alt: false });
|
|
2410
|
+
continue;
|
|
2411
|
+
}
|
|
2412
|
+
if (c === "|") {
|
|
2413
|
+
const g = top();
|
|
2414
|
+
if (g)
|
|
2415
|
+
g.alt = true;
|
|
2416
|
+
continue;
|
|
2417
|
+
}
|
|
2418
|
+
if (c === ")") {
|
|
2419
|
+
const g = open.pop() ?? { repeats: false, alt: false };
|
|
2420
|
+
const repeated = repeatsAt(p, i + 1);
|
|
2421
|
+
if (repeated && (g.repeats || g.alt))
|
|
2422
|
+
return true;
|
|
2423
|
+
const parent = top();
|
|
2424
|
+
if (parent) {
|
|
2425
|
+
if (g.repeats || unboundedAt(p, i + 1))
|
|
2426
|
+
parent.repeats = true;
|
|
2427
|
+
if (g.alt && !repeated)
|
|
2428
|
+
parent.alt = true;
|
|
2429
|
+
}
|
|
2430
|
+
continue;
|
|
2431
|
+
}
|
|
2432
|
+
const g = top();
|
|
2433
|
+
if (g && unboundedAt(p, i))
|
|
2434
|
+
g.repeats = true;
|
|
2327
2435
|
}
|
|
2328
|
-
|
|
2436
|
+
return false;
|
|
2437
|
+
}
|
|
2438
|
+
function priceFromPattern(body, pattern) {
|
|
2439
|
+
if (patternProblem(pattern) !== null)
|
|
2440
|
+
return null;
|
|
2441
|
+
const re = new RegExp(pattern, "i");
|
|
2442
|
+
const m = re.exec(body.slice(0, MAX_PATTERN_BODY_CHARS));
|
|
2329
2443
|
const captured = m?.[1] ?? m?.[0];
|
|
2330
2444
|
if (captured === undefined)
|
|
2331
2445
|
return null;
|
|
@@ -2355,8 +2469,19 @@ function extractPrice(body, pattern) {
|
|
|
2355
2469
|
}
|
|
2356
2470
|
|
|
2357
2471
|
// node_modules/@rackbops/docket-types/dist/money.js
|
|
2472
|
+
var ZERO_DECIMAL = new Set("BIF CLP DJF GNF ISK JPY KMF KRW PYG RWF UGX UYI VND VUV XAF XOF XPF".split(" "));
|
|
2473
|
+
var THREE_DECIMAL = new Set("BHD IQD JOD KWD LYD OMR TND".split(" "));
|
|
2474
|
+
function currencyCode(raw) {
|
|
2475
|
+
const c = (raw ?? "").trim().toUpperCase();
|
|
2476
|
+
return /^[A-Z]{3}$/.test(c) ? c : undefined;
|
|
2477
|
+
}
|
|
2478
|
+
function minorUnits(currency) {
|
|
2479
|
+
const c = currency.trim().toUpperCase();
|
|
2480
|
+
return ZERO_DECIMAL.has(c) ? 1 : THREE_DECIMAL.has(c) ? 1000 : 100;
|
|
2481
|
+
}
|
|
2358
2482
|
function money(amount, currency) {
|
|
2359
|
-
|
|
2483
|
+
const digits = Math.round(Math.log10(minorUnits(currency)));
|
|
2484
|
+
return `${amount.toFixed(digits)} ${currency}`.trim();
|
|
2360
2485
|
}
|
|
2361
2486
|
|
|
2362
2487
|
// node_modules/@rackbops/docket-types/dist/price.js
|
|
@@ -2382,7 +2507,8 @@ function stateOf(value) {
|
|
|
2382
2507
|
peak: s.peak ?? null,
|
|
2383
2508
|
below: s.below ?? false,
|
|
2384
2509
|
misses: s.misses,
|
|
2385
|
-
alerts: s.alerts ?? 0
|
|
2510
|
+
alerts: s.alerts ?? 0,
|
|
2511
|
+
...typeof s.currency === "string" && s.currency !== "" ? { currency: s.currency } : {}
|
|
2386
2512
|
};
|
|
2387
2513
|
}
|
|
2388
2514
|
function dropPercent(config) {
|
|
@@ -2441,6 +2567,9 @@ var price = defineTaskType({
|
|
|
2441
2567
|
if (!fetch2)
|
|
2442
2568
|
throw new Error("the price type needs the Fetch port; the host configured none");
|
|
2443
2569
|
const state = stateOf(ctx.state);
|
|
2570
|
+
const refused = ctx.config.pattern ? patternProblem(ctx.config.pattern) : null;
|
|
2571
|
+
if (refused)
|
|
2572
|
+
return miss(ctx, state, `pattern refused: ${refused}`);
|
|
2444
2573
|
let body;
|
|
2445
2574
|
try {
|
|
2446
2575
|
const response = await fetch2.get(ctx.config.url);
|
|
@@ -2458,12 +2587,14 @@ var price = defineTaskType({
|
|
|
2458
2587
|
const seen = found.value;
|
|
2459
2588
|
const currency = found.currency ?? ctx.config.currency ?? "";
|
|
2460
2589
|
const point = currency ? { value: seen, unit: currency, note: "observed" } : { value: seen, note: "observed" };
|
|
2590
|
+
const code = currencyCode(currency) ?? "";
|
|
2591
|
+
const kept = code === "" ? {} : { currency: code };
|
|
2461
2592
|
const rule = baselineRule(ctx.config);
|
|
2462
2593
|
const drop = dropPercent(ctx.config);
|
|
2463
2594
|
const ref = reference(state, rule);
|
|
2464
2595
|
if (ref === null) {
|
|
2465
2596
|
return {
|
|
2466
|
-
state: { first: seen, last: seen, peak: seen, below: false, misses: 0, alerts: 0 },
|
|
2597
|
+
state: { first: seen, last: seen, peak: seen, below: false, misses: 0, alerts: 0, ...kept },
|
|
2467
2598
|
series: [point],
|
|
2468
2599
|
notify: {
|
|
2469
2600
|
text: `Now tracking ${ctx.task.title} at ${money(seen, currency)}. I will say when it ` + `drops ${drop}% or more from the ${rule} price seen. ${ctx.config.url}`,
|
|
@@ -2472,6 +2603,11 @@ var price = defineTaskType({
|
|
|
2472
2603
|
summary: `baseline ${money(seen, currency)}`
|
|
2473
2604
|
};
|
|
2474
2605
|
}
|
|
2606
|
+
const declared = currencyCode(found.currency) ?? "";
|
|
2607
|
+
const known = state.currency ?? currencyCode(ctx.config.currency) ?? "";
|
|
2608
|
+
if (known !== "" && declared !== "" && declared !== known) {
|
|
2609
|
+
return miss(ctx, state, `the page now prices in ${declared}, not ${known}`);
|
|
2610
|
+
}
|
|
2475
2611
|
const line = ref * (1 - drop / 100);
|
|
2476
2612
|
const below = seen <= line + 0.000000001;
|
|
2477
2613
|
const crossing = below && !state.below;
|
|
@@ -2481,7 +2617,8 @@ var price = defineTaskType({
|
|
|
2481
2617
|
peak: Math.max(state.peak ?? seen, seen),
|
|
2482
2618
|
below,
|
|
2483
2619
|
misses: 0,
|
|
2484
|
-
alerts: state.alerts + (crossing ? 1 : 0)
|
|
2620
|
+
alerts: state.alerts + (crossing ? 1 : 0),
|
|
2621
|
+
...state.currency ? { currency: state.currency } : kept
|
|
2485
2622
|
};
|
|
2486
2623
|
if (!crossing) {
|
|
2487
2624
|
return {
|
|
@@ -2513,8 +2650,9 @@ var DEFAULT_SNOOZE_MS = 60 * 60000;
|
|
|
2513
2650
|
function snoozeUntil(payload, now) {
|
|
2514
2651
|
if (typeof payload === "object" && payload !== null) {
|
|
2515
2652
|
const p = payload;
|
|
2516
|
-
|
|
2517
|
-
|
|
2653
|
+
const until = typeof p.until === "string" ? Date.parse(p.until) : Number.NaN;
|
|
2654
|
+
if (!Number.isNaN(until) && until > now.getTime())
|
|
2655
|
+
return new Date(until);
|
|
2518
2656
|
if (typeof p.minutes === "number" && Number.isFinite(p.minutes) && p.minutes > 0) {
|
|
2519
2657
|
return new Date(now.getTime() + p.minutes * 60000);
|
|
2520
2658
|
}
|
|
@@ -2761,7 +2899,10 @@ function safeUrl(value) {
|
|
|
2761
2899
|
if (url.username || url.password)
|
|
2762
2900
|
return null;
|
|
2763
2901
|
const text = url.toString();
|
|
2764
|
-
|
|
2902
|
+
const decoded = decodeURIComponentSafe(text);
|
|
2903
|
+
if (/@(everyone|here)\b|<[@#]/i.test(decoded))
|
|
2904
|
+
return null;
|
|
2905
|
+
if (decoded.includes("]("))
|
|
2765
2906
|
return null;
|
|
2766
2907
|
return text;
|
|
2767
2908
|
}
|
|
@@ -2783,7 +2924,7 @@ function parseAnswer(value) {
|
|
|
2783
2924
|
if (summary.length === 0)
|
|
2784
2925
|
return null;
|
|
2785
2926
|
const findings = [];
|
|
2786
|
-
const
|
|
2927
|
+
const unsourced = [];
|
|
2787
2928
|
for (const f of value.findings) {
|
|
2788
2929
|
if (!isObj2(f) || typeof f.claim !== "string")
|
|
2789
2930
|
continue;
|
|
@@ -2793,10 +2934,11 @@ function parseAnswer(value) {
|
|
|
2793
2934
|
const raw = Array.isArray(f.sources) ? f.sources : [];
|
|
2794
2935
|
const sources = [...new Set(raw.map(safeUrl).filter((u) => u !== null))];
|
|
2795
2936
|
if (sources.length === 0)
|
|
2796
|
-
|
|
2937
|
+
unsourced.push(`${claim} (no source given)`);
|
|
2797
2938
|
else
|
|
2798
2939
|
findings.push({ claim, sources: sources.slice(0, MAX_SOURCES_PER_FINDING) });
|
|
2799
2940
|
}
|
|
2941
|
+
const uncertain = [];
|
|
2800
2942
|
const said = Array.isArray(value.uncertain) ? value.uncertain : [];
|
|
2801
2943
|
for (const u of said) {
|
|
2802
2944
|
if (typeof u !== "string")
|
|
@@ -2805,6 +2947,7 @@ function parseAnswer(value) {
|
|
|
2805
2947
|
if (line.length > 0)
|
|
2806
2948
|
uncertain.push(line);
|
|
2807
2949
|
}
|
|
2950
|
+
uncertain.push(...unsourced);
|
|
2808
2951
|
const blocked = parseBlocked(value.blocked);
|
|
2809
2952
|
return {
|
|
2810
2953
|
summary,
|
|
@@ -2923,106 +3066,327 @@ function renderUpdate(title, update, review) {
|
|
|
2923
3066
|
${note}`;
|
|
2924
3067
|
}
|
|
2925
3068
|
|
|
2926
|
-
// node_modules/@rackbops/docket-types/dist/
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
var MAX_GATHERED_CHARS = 1e5;
|
|
2931
|
-
var BARRED_SITES = [
|
|
2932
|
-
{
|
|
2933
|
-
site: "boardgamegeek.com",
|
|
2934
|
-
domains: ["boardgamegeek.com", "geekdo.com"],
|
|
2935
|
-
host: /(^|\.)(boardgamegeek|geekdo)\.com$/i,
|
|
2936
|
-
names: "board\\s?game\\s?geek|bgg|geekdo",
|
|
2937
|
-
everywhere: false
|
|
2938
|
-
},
|
|
2939
|
-
{
|
|
2940
|
-
site: "ebay.com",
|
|
2941
|
-
domains: ["ebay.*", "ebay.co.*", "ebay.com.*"],
|
|
2942
|
-
host: /(^|\.)ebay(\.[a-z]{2,3}){1,2}$/i,
|
|
2943
|
-
names: "ebay",
|
|
2944
|
-
everywhere: true
|
|
2945
|
-
}
|
|
2946
|
-
];
|
|
2947
|
-
function namedIn(b) {
|
|
2948
|
-
return new RegExp(`\\b(?:${b.names})\\b`, "i");
|
|
3069
|
+
// node_modules/@rackbops/docket-types/dist/want-sources.js
|
|
3070
|
+
import { createHash } from "crypto";
|
|
3071
|
+
class SourceMiss extends Error {
|
|
3072
|
+
name = "SourceMiss";
|
|
2949
3073
|
}
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
var FAILED_AFTER = "(?:refused|blocked|(?:was|were) blocked|returned (?:a |an )?(?:403|error)|403|forbidden|" + "showed a captcha|captcha|required (?:a )?sign-?in|inaccessible|unavailable|unreachable|" + "failed to (?:load|open)|timed out|(?:access )?(?:was )?denied|" + "(?:could|can|did|would|was|were|is|are|has|have|had) ?n[o']?t (?:be(?:en)? )?" + "(?:checked|opened|read|accessed|reached|loaded|load|open|consulted|used|available|reachable|accessible|" + "looked at)|not (?:been )?(?:checked|opened|read|accessed|reached|loaded|consulted|" + "available|reachable|accessible|looked at))\\b";
|
|
2954
|
-
function unreadShapes(b) {
|
|
2955
|
-
const site = `(?:${b.names})(?:'s)?[,):]?`;
|
|
2956
|
-
return [
|
|
2957
|
-
new RegExp(`\\b${NEG_BEFORE}${READ_VERB}\\s+${GAP}${site}`, "i"),
|
|
2958
|
-
new RegExp(`\\b${site}\\s+${GAP}${FAILED_AFTER}`, "i"),
|
|
2959
|
-
new RegExp(`\\bno (?:${b.names})(?:'s)? (?:data|ratings?|rankings?|ranks?|scores?|results?)\\b`, "i")
|
|
2960
|
-
];
|
|
3074
|
+
|
|
3075
|
+
class SourceUnavailableError extends SourceMiss {
|
|
3076
|
+
name = "SourceUnavailableError";
|
|
2961
3077
|
}
|
|
2962
|
-
|
|
2963
|
-
|
|
3078
|
+
var MAX_LISTINGS = 100;
|
|
3079
|
+
var TITLE_CHARS = 150;
|
|
3080
|
+
var CONDITION_CHARS = 40;
|
|
3081
|
+
var SELLER_CHARS = 80;
|
|
3082
|
+
function checkSearchStatus(status, who, refused, unavailable = {}) {
|
|
3083
|
+
const said = unavailable[status] ?? (status === 401 || status === 403 ? refused : undefined);
|
|
3084
|
+
if (said !== undefined)
|
|
3085
|
+
throw new SourceUnavailableError(said);
|
|
3086
|
+
if (status === 429 || status >= 500)
|
|
3087
|
+
throw new SourceMiss(`${who} is busy (HTTP ${status})`);
|
|
3088
|
+
if (status !== 200)
|
|
3089
|
+
throw new SourceMiss(`${who} answered HTTP ${status}`);
|
|
2964
3090
|
}
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
return
|
|
3091
|
+
var MAX_DEPTH = 8;
|
|
3092
|
+
function listingKey(id) {
|
|
3093
|
+
return createHash("sha256").update(id).digest("hex").slice(0, 32);
|
|
2968
3094
|
}
|
|
2969
|
-
|
|
2970
|
-
|
|
3095
|
+
var NEVER_EBAY = "x-tracker-never-ebay";
|
|
3096
|
+
var EBAY_HOST = /(^|\.)ebay\.[a-z]{2,3}(\.[a-z]{2})?$/i;
|
|
3097
|
+
function isEbayHost(host) {
|
|
3098
|
+
return EBAY_HOST.test(host.replace(/\.$/, ""));
|
|
2971
3099
|
}
|
|
2972
|
-
var
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
try {
|
|
2978
|
-
return new URL(url).hostname;
|
|
2979
|
-
} catch {
|
|
2980
|
-
return "";
|
|
2981
|
-
}
|
|
3100
|
+
var isObj3 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
3101
|
+
function typesOf(node) {
|
|
3102
|
+
const t = node["@type"];
|
|
3103
|
+
const all = Array.isArray(t) ? t : [t];
|
|
3104
|
+
return all.filter((x) => typeof x === "string").map((x) => x.replace(/^https?:\/\/schema\.org\//, ""));
|
|
2982
3105
|
}
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
3106
|
+
var PRODUCT_TYPES = new Set([
|
|
3107
|
+
"Product",
|
|
3108
|
+
"IndividualProduct",
|
|
3109
|
+
"ProductModel",
|
|
3110
|
+
"ProductGroup",
|
|
3111
|
+
"Book",
|
|
3112
|
+
"Game",
|
|
3113
|
+
"VideoGame",
|
|
3114
|
+
"Vehicle"
|
|
3115
|
+
]);
|
|
3116
|
+
function text(v, max) {
|
|
3117
|
+
if (typeof v === "string")
|
|
3118
|
+
return clean(v, max, true);
|
|
3119
|
+
if (isObj3(v) && typeof v.name === "string")
|
|
3120
|
+
return clean(v.name, max, true);
|
|
3121
|
+
return "";
|
|
2989
3122
|
}
|
|
2990
|
-
function
|
|
3123
|
+
function conditionOf(v) {
|
|
3124
|
+
const raw = text(v, 200);
|
|
3125
|
+
const m = /(New|Used|Refurbished|Damaged)Condition$/i.exec(raw);
|
|
3126
|
+
return m ? (m[1] ?? "").toLowerCase() : clean(raw, CONDITION_CHARS, true);
|
|
3127
|
+
}
|
|
3128
|
+
function firstOffer(offers) {
|
|
3129
|
+
if (Array.isArray(offers))
|
|
3130
|
+
return offers.find(isObj3) ?? null;
|
|
3131
|
+
return isObj3(offers) ? offers : null;
|
|
3132
|
+
}
|
|
3133
|
+
var PER_VIEW = /^(utm_[a-z]+|_pos|_sid|_ss|_psq|srsltid|gclid|gbraid|wbraid|fbclid|msclkid|mc_cid|mc_eid|pf_rd_[a-z]+|pd_rd_[a-z]+)$/i;
|
|
3134
|
+
function withoutPerView(u) {
|
|
3135
|
+
u.hash = "";
|
|
3136
|
+
for (const key of [...u.searchParams.keys()])
|
|
3137
|
+
if (PER_VIEW.test(key))
|
|
3138
|
+
u.searchParams.delete(key);
|
|
3139
|
+
return u;
|
|
3140
|
+
}
|
|
3141
|
+
function absolute(raw, base) {
|
|
3142
|
+
if (typeof raw !== "string" || raw.trim() === "")
|
|
3143
|
+
return null;
|
|
2991
3144
|
try {
|
|
2992
|
-
|
|
2993
|
-
return `${siteOf(u.hostname)}${u.pathname.replace(/\/+$/, "")}`;
|
|
3145
|
+
return safeUrl(withoutPerView(new URL(raw.trim(), base)).toString());
|
|
2994
3146
|
} catch {
|
|
2995
|
-
return
|
|
3147
|
+
return null;
|
|
2996
3148
|
}
|
|
2997
3149
|
}
|
|
2998
|
-
function
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3150
|
+
function listingOf(node, base, fallbackUrl) {
|
|
3151
|
+
const title = text(node.name, TITLE_CHARS);
|
|
3152
|
+
const offer = firstOffer(node.offers);
|
|
3153
|
+
const url = absolute(node.url, base) ?? absolute(fallbackUrl, base) ?? absolute(offer?.url, base);
|
|
3154
|
+
if (!title || !url)
|
|
3155
|
+
return null;
|
|
3156
|
+
const price = priceFromJson(node.offers ?? null);
|
|
3157
|
+
const condition = conditionOf(offer?.itemCondition ?? node.itemCondition);
|
|
3158
|
+
const seller = text(offer?.seller, SELLER_CHARS);
|
|
3159
|
+
return {
|
|
3160
|
+
id: url,
|
|
3161
|
+
title,
|
|
3162
|
+
url,
|
|
3163
|
+
...price ? { price: price.value } : {},
|
|
3164
|
+
...price?.currency ? { currency: price.currency } : {},
|
|
3165
|
+
...condition ? { condition } : {},
|
|
3166
|
+
...seller ? { seller } : {}
|
|
3167
|
+
};
|
|
3006
3168
|
}
|
|
3007
|
-
function
|
|
3169
|
+
function readListings(html, pageUrl) {
|
|
3170
|
+
let lists = 0;
|
|
3008
3171
|
const out = [];
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
if (
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3172
|
+
const seen = new Set;
|
|
3173
|
+
const add = (l) => {
|
|
3174
|
+
if (!l || seen.has(l.id) || out.length >= MAX_LISTINGS)
|
|
3175
|
+
return;
|
|
3176
|
+
seen.add(l.id);
|
|
3177
|
+
out.push(l);
|
|
3178
|
+
};
|
|
3179
|
+
const walk = (node, depth) => {
|
|
3180
|
+
if (depth > MAX_DEPTH || out.length >= MAX_LISTINGS)
|
|
3181
|
+
return;
|
|
3182
|
+
if (Array.isArray(node)) {
|
|
3183
|
+
for (const n of node)
|
|
3184
|
+
walk(n, depth + 1);
|
|
3185
|
+
return;
|
|
3186
|
+
}
|
|
3187
|
+
if (!isObj3(node))
|
|
3188
|
+
return;
|
|
3189
|
+
const types = typesOf(node);
|
|
3190
|
+
if (types.includes("ItemList")) {
|
|
3191
|
+
lists++;
|
|
3192
|
+
const elements = Array.isArray(node.itemListElement) ? node.itemListElement : [node.itemListElement];
|
|
3193
|
+
for (const el of elements) {
|
|
3194
|
+
if (!isObj3(el))
|
|
3195
|
+
continue;
|
|
3196
|
+
const item = isObj3(el.item) ? el.item : el;
|
|
3197
|
+
if (typesOf(item).includes("ItemList"))
|
|
3198
|
+
walk(item, depth + 1);
|
|
3199
|
+
else
|
|
3200
|
+
add(listingOf(item, pageUrl, el.url ?? (typeof el.item === "string" ? el.item : undefined)));
|
|
3201
|
+
}
|
|
3202
|
+
} else if (types.some((t) => PRODUCT_TYPES.has(t))) {
|
|
3203
|
+
add(listingOf(node, pageUrl, pageUrl));
|
|
3204
|
+
}
|
|
3205
|
+
if (node["@graph"] !== undefined)
|
|
3206
|
+
walk(node["@graph"], depth + 1);
|
|
3207
|
+
if (isObj3(node.mainEntity))
|
|
3208
|
+
walk(node.mainEntity, depth + 1);
|
|
3209
|
+
};
|
|
3210
|
+
for (const block of jsonLdBlocks(html))
|
|
3211
|
+
walk(block, 0);
|
|
3212
|
+
return { listings: out, lists };
|
|
3213
|
+
}
|
|
3214
|
+
var pageSource = {
|
|
3215
|
+
id: "page",
|
|
3216
|
+
async search(target, fetch2) {
|
|
3217
|
+
if (!fetch2)
|
|
3218
|
+
throw new SourceMiss("this bot reads no pages");
|
|
3219
|
+
let body;
|
|
3220
|
+
try {
|
|
3221
|
+
const response = await fetch2.get(target, { [NEVER_EBAY]: "1" });
|
|
3222
|
+
if (response.status !== 200)
|
|
3223
|
+
throw new SourceMiss(`HTTP ${response.status}`);
|
|
3224
|
+
body = response.body;
|
|
3225
|
+
} catch (err) {
|
|
3226
|
+
if (err instanceof SourceMiss)
|
|
3227
|
+
throw err;
|
|
3228
|
+
throw new SourceMiss(`the read failed: ${err instanceof Error ? err.message : String(err)}`, {
|
|
3229
|
+
cause: err
|
|
3230
|
+
});
|
|
3231
|
+
}
|
|
3232
|
+
const { listings, lists } = readListings(body, target);
|
|
3233
|
+
if (listings.length === 0 && lists === 0)
|
|
3234
|
+
throw new SourceMiss("no listings in the page's structured data");
|
|
3235
|
+
return listings;
|
|
3236
|
+
}
|
|
3237
|
+
};
|
|
3238
|
+
var nonNegative = (n) => n !== null && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
3239
|
+
function ebayItem(url) {
|
|
3240
|
+
if (url === null)
|
|
3241
|
+
return null;
|
|
3242
|
+
const u = new URL(url);
|
|
3243
|
+
const item = /^\/itm\/(?:[^/]+\/)?(\d{6,20})\/?$/.exec(u.pathname);
|
|
3244
|
+
return isEbayHost(u.hostname) && item ? `https://${u.hostname}/itm/${item[1]}` : url;
|
|
3245
|
+
}
|
|
3246
|
+
function submittedListing(raw) {
|
|
3247
|
+
const title = typeof raw.title === "string" ? clean(raw.title, TITLE_CHARS, true) : "";
|
|
3248
|
+
if (typeof raw.url !== "string" || !/^https?:\/\/[^/\\]/i.test(raw.url.trim()))
|
|
3249
|
+
return null;
|
|
3250
|
+
const url = ebayItem(absolute(raw.url, "https://invalid.example/"));
|
|
3251
|
+
if (!title || !url || url.startsWith("https://invalid.example/"))
|
|
3252
|
+
return null;
|
|
3253
|
+
const price = typeof raw.price === "number" && Number.isFinite(raw.price) && raw.price >= 0 ? raw.price : typeof raw.price === "string" ? nonNegative(parsePrice(raw.price)) : undefined;
|
|
3254
|
+
const currency = typeof raw.currency === "string" && /^[A-Za-z]{3}$/.test(raw.currency.trim()) ? raw.currency.trim().toUpperCase() : undefined;
|
|
3255
|
+
const condition = typeof raw.condition === "string" ? clean(raw.condition, CONDITION_CHARS, true) : "";
|
|
3256
|
+
const seller = typeof raw.seller === "string" ? clean(raw.seller, SELLER_CHARS, true) : "";
|
|
3257
|
+
return {
|
|
3258
|
+
id: url,
|
|
3259
|
+
title,
|
|
3260
|
+
url,
|
|
3261
|
+
...price !== undefined ? { price } : {},
|
|
3262
|
+
...currency ? { currency } : {},
|
|
3263
|
+
...condition ? { condition } : {},
|
|
3264
|
+
...seller ? { seller } : {}
|
|
3265
|
+
};
|
|
3266
|
+
}
|
|
3267
|
+
function inboxSource(read) {
|
|
3268
|
+
return {
|
|
3269
|
+
id: "inbox",
|
|
3270
|
+
async search(target) {
|
|
3271
|
+
let items;
|
|
3272
|
+
try {
|
|
3273
|
+
items = await read(target);
|
|
3274
|
+
} catch (err) {
|
|
3275
|
+
throw new SourceMiss("the inbox could not be read", { cause: err });
|
|
3276
|
+
}
|
|
3277
|
+
const byId = new Map;
|
|
3278
|
+
for (const raw of items) {
|
|
3279
|
+
const l = submittedListing(raw);
|
|
3280
|
+
if (!l)
|
|
3281
|
+
continue;
|
|
3282
|
+
byId.delete(l.id);
|
|
3283
|
+
byId.set(l.id, l);
|
|
3284
|
+
}
|
|
3285
|
+
return [...byId.values()].slice(-MAX_LISTINGS);
|
|
3286
|
+
}
|
|
3287
|
+
};
|
|
3288
|
+
}
|
|
3289
|
+
|
|
3290
|
+
// node_modules/@rackbops/docket-types/dist/research-sources.js
|
|
3291
|
+
var MAX_GATHERED_PAGES = 10;
|
|
3292
|
+
var MAX_PAGE_TITLE_CHARS = 200;
|
|
3293
|
+
var MAX_PAGE_CHARS = 12000;
|
|
3294
|
+
var MAX_GATHERED_CHARS = 1e5;
|
|
3295
|
+
var BARRED_SITES = [
|
|
3296
|
+
{
|
|
3297
|
+
site: "boardgamegeek.com",
|
|
3298
|
+
domains: ["boardgamegeek.com", "geekdo.com"],
|
|
3299
|
+
host: /(^|\.)(boardgamegeek|geekdo)\.com$/i,
|
|
3300
|
+
names: "board\\s?game\\s?geek|bgg|geekdo",
|
|
3301
|
+
everywhere: false
|
|
3302
|
+
},
|
|
3303
|
+
{
|
|
3304
|
+
site: "ebay.com",
|
|
3305
|
+
domains: ["ebay.*", "ebay.co.*", "ebay.com.*"],
|
|
3306
|
+
host: EBAY_HOST,
|
|
3307
|
+
names: "ebay",
|
|
3308
|
+
everywhere: true
|
|
3309
|
+
}
|
|
3310
|
+
];
|
|
3311
|
+
function namedIn(b) {
|
|
3312
|
+
return new RegExp(`\\b(?:${b.names})\\b`, "i");
|
|
3313
|
+
}
|
|
3314
|
+
var GAP = "(?:[\\w'-]+[,):]?\\s+){0,3}";
|
|
3315
|
+
var READ_VERB = "(?:(?:check|open|read|access|reach|load|consult|get|visit|fetch|look(?:\\s+at)?)" + "(?:ed|ing|s)?|verif(?:y|ies|ied|ying)|retriev(?:e|es|ed|ing))";
|
|
3316
|
+
var NEG_BEFORE = "(?:(?:did|could|can|would|should|have|has|had|do|does) ?n[o']?t|cannot|(?:was|were) ?n[o']?t able to|" + "unable to|never|not|without)\\s+(?:be(?:en)?\\s+able\\s+to\\s+)?";
|
|
3317
|
+
var FAILED_AFTER = "(?:refused|blocked|(?:was|were) blocked|returned (?:a |an )?(?:403|error)|403|forbidden|" + "showed a captcha|captcha|required (?:a )?sign-?in|inaccessible|unavailable|unreachable|" + "failed to (?:load|open)|timed out|(?:access )?(?:was )?denied|" + "(?:could|can|did|would|was|were|is|are|has|have|had) ?n[o']?t (?:be(?:en)? )?" + "(?:checked|opened|read|accessed|reached|loaded|load|open|consulted|used|available|reachable|accessible|" + "looked at)|not (?:been )?(?:checked|opened|read|accessed|reached|loaded|consulted|" + "available|reachable|accessible|looked at))\\b";
|
|
3318
|
+
function unreadShapes(b) {
|
|
3319
|
+
const site = `(?:${b.names})(?:'s)?[,):]?`;
|
|
3320
|
+
return [
|
|
3321
|
+
new RegExp(`\\b${NEG_BEFORE}${READ_VERB}\\s+${GAP}${site}`, "i"),
|
|
3322
|
+
new RegExp(`\\b${site}\\s+${GAP}${FAILED_AFTER}`, "i"),
|
|
3323
|
+
new RegExp(`\\bno (?:${b.names})(?:'s)? (?:data|ratings?|rankings?|ranks?|scores?|results?)\\b`, "i")
|
|
3324
|
+
];
|
|
3325
|
+
}
|
|
3326
|
+
function sentences(text) {
|
|
3327
|
+
return text.replace(/[\u2018\u2019]/g, "'").replace(/\b(e)\.(g)\.|\b(i)\.(e)\./gi, (_m, a, b, c, d) => `${a ?? c}${b ?? d}`).split(/(?<=[.!?;])\s+|\n/);
|
|
3328
|
+
}
|
|
3329
|
+
function saysUnread(b, lines) {
|
|
3330
|
+
const shapes = unreadShapes(b);
|
|
3331
|
+
return lines.flatMap(sentences).some((line) => shapes.some((r) => r.test(line)));
|
|
3332
|
+
}
|
|
3333
|
+
function barredDomains(everywhereOnly = false) {
|
|
3334
|
+
return BARRED_SITES.filter((b) => !everywhereOnly || b.everywhere).flatMap((b) => b.domains.flatMap((d) => [d, `*.${d}`]));
|
|
3335
|
+
}
|
|
3336
|
+
var SENT_DISALLOWED_TOOLS = barredDomains().map((host) => `WebFetch(domain:${host})`);
|
|
3337
|
+
var GATHERED_ALLOWED_TOOLS = ["WebSearch"];
|
|
3338
|
+
var GATHERED_DENIED_TOOLS = ["WebFetch"];
|
|
3339
|
+
var WEB_DISALLOWED_TOOLS = barredDomains(true).map((host) => `WebFetch(domain:${host})`);
|
|
3340
|
+
function hostOf(url) {
|
|
3341
|
+
try {
|
|
3342
|
+
return new URL(url).hostname.replace(/\.$/, "");
|
|
3343
|
+
} catch {
|
|
3344
|
+
return "";
|
|
3345
|
+
}
|
|
3346
|
+
}
|
|
3347
|
+
function siteOf(text) {
|
|
3348
|
+
const t = text.trim().toLowerCase().replace(/^www\./, "").replace(/\.$/, "");
|
|
3349
|
+
for (const b of BARRED_SITES)
|
|
3350
|
+
if (b.host.test(t) || namedIn(b).test(t))
|
|
3351
|
+
return b.site;
|
|
3352
|
+
return t;
|
|
3353
|
+
}
|
|
3354
|
+
function pageOf(url) {
|
|
3355
|
+
try {
|
|
3356
|
+
const u = new URL(url);
|
|
3357
|
+
return `${siteOf(u.hostname)}${u.pathname.replace(/\/+$/, "")}`;
|
|
3358
|
+
} catch {
|
|
3359
|
+
return "";
|
|
3360
|
+
}
|
|
3361
|
+
}
|
|
3362
|
+
function blockedKey(entry) {
|
|
3363
|
+
return entry.url ? pageOf(entry.url) : siteOf(entry.site);
|
|
3364
|
+
}
|
|
3365
|
+
function sitesOf(entry) {
|
|
3366
|
+
const out = new Set([siteOf(entry.site)]);
|
|
3367
|
+
if (entry.url)
|
|
3368
|
+
out.add(siteOf(hostOf(entry.url)));
|
|
3369
|
+
return out;
|
|
3370
|
+
}
|
|
3371
|
+
function uniqueBlocked(entries) {
|
|
3372
|
+
const out = [];
|
|
3373
|
+
for (const e of entries) {
|
|
3374
|
+
const key = blockedKey(e);
|
|
3375
|
+
if (out.some((o) => o.url && blockedKey(o) === key))
|
|
3376
|
+
continue;
|
|
3377
|
+
if (!e.url) {
|
|
3378
|
+
const site = siteOf(e.site);
|
|
3379
|
+
if (out.some((o) => sitesOf(o).has(site)))
|
|
3380
|
+
continue;
|
|
3381
|
+
} else {
|
|
3382
|
+
const sites = sitesOf(e);
|
|
3383
|
+
const bare = out.findIndex((o) => !o.url && sites.has(siteOf(o.site)));
|
|
3384
|
+
if (bare >= 0) {
|
|
3385
|
+
out[bare] = e;
|
|
3386
|
+
continue;
|
|
3387
|
+
}
|
|
3388
|
+
}
|
|
3389
|
+
if (out.length >= MAX_BLOCKED)
|
|
3026
3390
|
break;
|
|
3027
3391
|
out.push(e);
|
|
3028
3392
|
}
|
|
@@ -3154,27 +3518,36 @@ function categoryChecks() {
|
|
|
3154
3518
|
];
|
|
3155
3519
|
}
|
|
3156
3520
|
|
|
3521
|
+
// node_modules/@rackbops/docket-types/dist/retry.js
|
|
3522
|
+
var MAX_TRIES = 2;
|
|
3523
|
+
var AUTH_RETRY_MS = 3600000;
|
|
3524
|
+
var RETRIED = new Set([
|
|
3525
|
+
"schema_miss",
|
|
3526
|
+
"malformed",
|
|
3527
|
+
"timeout",
|
|
3528
|
+
"error",
|
|
3529
|
+
"auth_failed"
|
|
3530
|
+
]);
|
|
3531
|
+
function retryOutcome(kind, failures, now) {
|
|
3532
|
+
if (!RETRIED.has(kind) || failures >= MAX_TRIES)
|
|
3533
|
+
return null;
|
|
3534
|
+
const wait = kind === "auth_failed" ? AUTH_RETRY_MS : 0;
|
|
3535
|
+
const at = new Date(now.getTime() + wait).toISOString();
|
|
3536
|
+
return { at, followUp: wait > 0 ? { at } : {} };
|
|
3537
|
+
}
|
|
3538
|
+
|
|
3157
3539
|
// node_modules/@rackbops/docket-types/dist/research.js
|
|
3158
3540
|
function usesSentSources(config) {
|
|
3159
3541
|
return typeof config.sources === "string" && config.sources.trim().toLowerCase() === "sent";
|
|
3160
3542
|
}
|
|
3161
3543
|
var MAX_QUESTION_CHARS = 1000;
|
|
3162
3544
|
var MAX_CONTEXT_CHARS = 2000;
|
|
3163
|
-
var MAX_TRIES = 2;
|
|
3164
|
-
var AUTH_RETRY_MS = 3600000;
|
|
3165
3545
|
var RESEARCH_MAX_TURNS = 15;
|
|
3166
3546
|
var RESEARCH_MAX_BUDGET_USD = 1;
|
|
3167
3547
|
var RESEARCH_TIMEOUT_MS = 600000;
|
|
3168
3548
|
var REVIEW_MAX_TURNS = 8;
|
|
3169
3549
|
var REVIEW_MAX_BUDGET_USD = 0.5;
|
|
3170
3550
|
var REVIEW_TIMEOUT_MS = 300000;
|
|
3171
|
-
var RETRIED = new Set([
|
|
3172
|
-
"schema_miss",
|
|
3173
|
-
"malformed",
|
|
3174
|
-
"timeout",
|
|
3175
|
-
"error",
|
|
3176
|
-
"auth_failed"
|
|
3177
|
-
]);
|
|
3178
3551
|
var FRESH2 = { phase: "research", failures: 0, draft: null };
|
|
3179
3552
|
function passFields(s) {
|
|
3180
3553
|
const blocked = parseBlocked(s.blocked);
|
|
@@ -3392,14 +3765,9 @@ function failed(ctx, state, kind, detail) {
|
|
|
3392
3765
|
const phase = state.phase === "review" || state.phase === "awaiting-pages" ? state.phase : "research";
|
|
3393
3766
|
const failures = state.failures + 1;
|
|
3394
3767
|
const summary = clean(`${phase} ${kind} (try ${failures}): ${detail}`, 300, true);
|
|
3395
|
-
|
|
3396
|
-
|
|
3397
|
-
return {
|
|
3398
|
-
state: { ...state, phase, failures },
|
|
3399
|
-
followUp: wait > 0 ? { at: new Date(ctx.now.getTime() + wait).toISOString() } : {},
|
|
3400
|
-
summary
|
|
3401
|
-
};
|
|
3402
|
-
}
|
|
3768
|
+
const retry = retryOutcome(kind, failures, ctx.now);
|
|
3769
|
+
if (retry)
|
|
3770
|
+
return { state: { ...state, phase, failures }, followUp: retry.followUp, summary };
|
|
3403
3771
|
const said = SAID[kind] ?? "something went wrong";
|
|
3404
3772
|
if (secondPass(state)) {
|
|
3405
3773
|
return stands(ctx, state, `the fuller pass could not finish (${said})`, summary);
|
|
@@ -3685,7 +4053,7 @@ function researchType(options = {}) {
|
|
|
3685
4053
|
var research = researchType();
|
|
3686
4054
|
|
|
3687
4055
|
// node_modules/@rackbops/docket-types/dist/scout.js
|
|
3688
|
-
import { createHash } from "crypto";
|
|
4056
|
+
import { createHash as createHash2 } from "crypto";
|
|
3689
4057
|
var LENSES = ["general", "birthday", "anniversary", "christmas"];
|
|
3690
4058
|
var LENS_TEXT = {
|
|
3691
4059
|
general: "no occasion in particular -- anything they would genuinely enjoy hearing about or owning",
|
|
@@ -3734,18 +4102,17 @@ var SCOUT_SCHEMA = {
|
|
|
3734
4102
|
},
|
|
3735
4103
|
required: ["items"]
|
|
3736
4104
|
};
|
|
3737
|
-
function
|
|
4105
|
+
function isObj4(v) {
|
|
3738
4106
|
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
3739
4107
|
}
|
|
3740
4108
|
function itemKey(url) {
|
|
3741
|
-
const u = new URL(url);
|
|
3742
|
-
u.
|
|
3743
|
-
return createHash("sha256").update(u.toString()).digest("hex").slice(0, 32);
|
|
4109
|
+
const u = withoutPerView(new URL(url));
|
|
4110
|
+
return createHash2("sha256").update(u.toString()).digest("hex").slice(0, 32);
|
|
3744
4111
|
}
|
|
3745
4112
|
function scoutState(value) {
|
|
3746
|
-
if (!
|
|
4113
|
+
if (!isObj4(value))
|
|
3747
4114
|
return { shown: [], failures: 0 };
|
|
3748
|
-
const shown = (Array.isArray(value.shown) ? value.shown : []).filter((s) =>
|
|
4115
|
+
const shown = (Array.isArray(value.shown) ? value.shown : []).filter((s) => isObj4(s) && typeof s.k === "string" && typeof s.t === "string").slice(-MAX_SHOWN);
|
|
3749
4116
|
const failures = typeof value.failures === "number" && value.failures >= 0 ? value.failures : 0;
|
|
3750
4117
|
return { shown, failures };
|
|
3751
4118
|
}
|
|
@@ -3785,11 +4152,11 @@ function scoutJob(config, state) {
|
|
|
3785
4152
|
};
|
|
3786
4153
|
}
|
|
3787
4154
|
function parseScout(value) {
|
|
3788
|
-
if (!
|
|
4155
|
+
if (!isObj4(value) || !Array.isArray(value.items))
|
|
3789
4156
|
return null;
|
|
3790
4157
|
const items = [];
|
|
3791
4158
|
for (const raw of value.items) {
|
|
3792
|
-
if (!
|
|
4159
|
+
if (!isObj4(raw) || typeof raw.title !== "string")
|
|
3793
4160
|
continue;
|
|
3794
4161
|
const url = safeUrl(raw.url);
|
|
3795
4162
|
const title = clean(raw.title, ITEM_TITLE_CHARS, true);
|
|
@@ -3825,13 +4192,6 @@ function renderScout(title, items, shortfall, repeats) {
|
|
|
3825
4192
|
lines.push("", `Fewer than ${MIN_ITEMS}: ${shortfall}`);
|
|
3826
4193
|
return fitMessage(lines);
|
|
3827
4194
|
}
|
|
3828
|
-
var RETRIED2 = new Set([
|
|
3829
|
-
"schema_miss",
|
|
3830
|
-
"malformed",
|
|
3831
|
-
"timeout",
|
|
3832
|
-
"error",
|
|
3833
|
-
"auth_failed"
|
|
3834
|
-
]);
|
|
3835
4195
|
var SAID2 = {
|
|
3836
4196
|
auth_failed: "the runner could not sign in",
|
|
3837
4197
|
turn_cap: "it needed more steps than one run is allowed",
|
|
@@ -3844,14 +4204,9 @@ var SAID2 = {
|
|
|
3844
4204
|
function failed2(ctx, state, kind, detail) {
|
|
3845
4205
|
const failures = state.failures + 1;
|
|
3846
4206
|
const summary = clean(`scout ${kind} (try ${failures}): ${detail}`, 300, true);
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
return {
|
|
3850
|
-
state: { ...state, failures },
|
|
3851
|
-
followUp: wait > 0 ? { at: new Date(ctx.now.getTime() + wait).toISOString() } : {},
|
|
3852
|
-
summary
|
|
3853
|
-
};
|
|
3854
|
-
}
|
|
4207
|
+
const retry = retryOutcome(kind, failures, ctx.now);
|
|
4208
|
+
if (retry)
|
|
4209
|
+
return { state: { ...state, failures }, followUp: retry.followUp, summary };
|
|
3855
4210
|
const narrower = kind === "turn_cap" || kind === "budget_cap" ? " Fewer or narrower interests may help." : "";
|
|
3856
4211
|
return {
|
|
3857
4212
|
state: { ...state, failures: 0 },
|
|
@@ -3901,252 +4256,43 @@ function afterRun(ctx, state, output) {
|
|
|
3901
4256
|
var scout = defineTaskType({
|
|
3902
4257
|
id: "scout",
|
|
3903
4258
|
lane: "execute",
|
|
3904
|
-
capabilities: ["notify"],
|
|
3905
|
-
schedule: ["calendar"],
|
|
3906
|
-
intake: {
|
|
3907
|
-
options: [
|
|
3908
|
-
{
|
|
3909
|
-
name: "interests",
|
|
3910
|
-
description: "What they are into, separated by commas",
|
|
3911
|
-
required: true,
|
|
3912
|
-
kind: "string"
|
|
3913
|
-
},
|
|
3914
|
-
{
|
|
3915
|
-
name: "lens",
|
|
3916
|
-
description: "The gift lens: general, birthday, anniversary or christmas",
|
|
3917
|
-
required: false,
|
|
3918
|
-
kind: "string"
|
|
3919
|
-
},
|
|
3920
|
-
{
|
|
3921
|
-
name: "for",
|
|
3922
|
-
description: "Who the ideas are for, if not you",
|
|
3923
|
-
required: false,
|
|
3924
|
-
kind: "string"
|
|
3925
|
-
},
|
|
3926
|
-
{ name: "notes", description: "Anything else to weigh", required: false, kind: "string" },
|
|
3927
|
-
{ name: "every", description: "Days between runs", required: false, kind: "integer" }
|
|
3928
|
-
]
|
|
3929
|
-
},
|
|
3930
|
-
async prepare(ctx) {
|
|
3931
|
-
return scoutJob(ctx.config, scoutState(ctx.state));
|
|
3932
|
-
},
|
|
3933
|
-
async finish(ctx, result) {
|
|
3934
|
-
const stored = scoutState(ctx.state);
|
|
3935
|
-
const state = ctx.occurrence.dedupeKey.startsWith(
|
|
3936
|
-
if (result.kind !== "success")
|
|
3937
|
-
return failed2(ctx, state, result.kind, result.detail);
|
|
3938
|
-
return afterRun(ctx, state, result.structuredOutput);
|
|
3939
|
-
}
|
|
3940
|
-
});
|
|
3941
|
-
|
|
3942
|
-
// node_modules/@rackbops/docket-types/dist/want-sources.js
|
|
3943
|
-
import { createHash as createHash2 } from "crypto";
|
|
3944
|
-
class SourceMiss extends Error {
|
|
3945
|
-
name = "SourceMiss";
|
|
3946
|
-
}
|
|
3947
|
-
|
|
3948
|
-
class SourceUnavailableError extends SourceMiss {
|
|
3949
|
-
name = "SourceUnavailableError";
|
|
3950
|
-
}
|
|
3951
|
-
var MAX_LISTINGS = 100;
|
|
3952
|
-
var TITLE_CHARS = 150;
|
|
3953
|
-
var CONDITION_CHARS = 40;
|
|
3954
|
-
var SELLER_CHARS = 80;
|
|
3955
|
-
var MAX_DEPTH = 8;
|
|
3956
|
-
function listingKey(id) {
|
|
3957
|
-
return createHash2("sha256").update(id).digest("hex").slice(0, 32);
|
|
3958
|
-
}
|
|
3959
|
-
var NEVER_EBAY = "x-tracker-never-ebay";
|
|
3960
|
-
function isEbayHost(host) {
|
|
3961
|
-
return /(^|\.)ebay\.[a-z]{2,3}(\.[a-z]{2})?$/i.test(host.replace(/\.$/, ""));
|
|
3962
|
-
}
|
|
3963
|
-
var isObj4 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
3964
|
-
function typesOf(node) {
|
|
3965
|
-
const t = node["@type"];
|
|
3966
|
-
const all = Array.isArray(t) ? t : [t];
|
|
3967
|
-
return all.filter((x) => typeof x === "string").map((x) => x.replace(/^https?:\/\/schema\.org\//, ""));
|
|
3968
|
-
}
|
|
3969
|
-
var PRODUCT_TYPES = new Set([
|
|
3970
|
-
"Product",
|
|
3971
|
-
"IndividualProduct",
|
|
3972
|
-
"ProductModel",
|
|
3973
|
-
"ProductGroup",
|
|
3974
|
-
"Book",
|
|
3975
|
-
"Game",
|
|
3976
|
-
"VideoGame",
|
|
3977
|
-
"Vehicle"
|
|
3978
|
-
]);
|
|
3979
|
-
function text(v, max) {
|
|
3980
|
-
if (typeof v === "string")
|
|
3981
|
-
return clean(v, max, true);
|
|
3982
|
-
if (isObj4(v) && typeof v.name === "string")
|
|
3983
|
-
return clean(v.name, max, true);
|
|
3984
|
-
return "";
|
|
3985
|
-
}
|
|
3986
|
-
function conditionOf(v) {
|
|
3987
|
-
const raw = text(v, 200);
|
|
3988
|
-
const m = /(New|Used|Refurbished|Damaged)Condition$/i.exec(raw);
|
|
3989
|
-
return m ? (m[1] ?? "").toLowerCase() : clean(raw, CONDITION_CHARS, true);
|
|
3990
|
-
}
|
|
3991
|
-
function firstOffer(offers) {
|
|
3992
|
-
if (Array.isArray(offers))
|
|
3993
|
-
return offers.find(isObj4) ?? null;
|
|
3994
|
-
return isObj4(offers) ? offers : null;
|
|
3995
|
-
}
|
|
3996
|
-
var PER_VIEW = /^(utm_[a-z]+|_pos|_sid|_ss|_psq|srsltid|gclid|gbraid|wbraid|fbclid|msclkid|mc_cid|mc_eid|pf_rd_[a-z]+|pd_rd_[a-z]+)$/i;
|
|
3997
|
-
function absolute(raw, base) {
|
|
3998
|
-
if (typeof raw !== "string" || raw.trim() === "")
|
|
3999
|
-
return null;
|
|
4000
|
-
try {
|
|
4001
|
-
const u = new URL(raw.trim(), base);
|
|
4002
|
-
u.hash = "";
|
|
4003
|
-
for (const key of [...u.searchParams.keys()])
|
|
4004
|
-
if (PER_VIEW.test(key))
|
|
4005
|
-
u.searchParams.delete(key);
|
|
4006
|
-
return safeUrl(u.toString());
|
|
4007
|
-
} catch {
|
|
4008
|
-
return null;
|
|
4009
|
-
}
|
|
4010
|
-
}
|
|
4011
|
-
function listingOf(node, base, fallbackUrl) {
|
|
4012
|
-
const title = text(node.name, TITLE_CHARS);
|
|
4013
|
-
const offer = firstOffer(node.offers);
|
|
4014
|
-
const url = absolute(node.url, base) ?? absolute(fallbackUrl, base) ?? absolute(offer?.url, base);
|
|
4015
|
-
if (!title || !url)
|
|
4016
|
-
return null;
|
|
4017
|
-
const price = priceFromJson(node.offers ?? null);
|
|
4018
|
-
const condition = conditionOf(offer?.itemCondition ?? node.itemCondition);
|
|
4019
|
-
const seller = text(offer?.seller, SELLER_CHARS);
|
|
4020
|
-
return {
|
|
4021
|
-
id: url,
|
|
4022
|
-
title,
|
|
4023
|
-
url,
|
|
4024
|
-
...price ? { price: price.value } : {},
|
|
4025
|
-
...price?.currency ? { currency: price.currency } : {},
|
|
4026
|
-
...condition ? { condition } : {},
|
|
4027
|
-
...seller ? { seller } : {}
|
|
4028
|
-
};
|
|
4029
|
-
}
|
|
4030
|
-
function readListings(html, pageUrl) {
|
|
4031
|
-
let lists = 0;
|
|
4032
|
-
const out = [];
|
|
4033
|
-
const seen = new Set;
|
|
4034
|
-
const add = (l) => {
|
|
4035
|
-
if (!l || seen.has(l.id) || out.length >= MAX_LISTINGS)
|
|
4036
|
-
return;
|
|
4037
|
-
seen.add(l.id);
|
|
4038
|
-
out.push(l);
|
|
4039
|
-
};
|
|
4040
|
-
const walk = (node, depth) => {
|
|
4041
|
-
if (depth > MAX_DEPTH || out.length >= MAX_LISTINGS)
|
|
4042
|
-
return;
|
|
4043
|
-
if (Array.isArray(node)) {
|
|
4044
|
-
for (const n of node)
|
|
4045
|
-
walk(n, depth + 1);
|
|
4046
|
-
return;
|
|
4047
|
-
}
|
|
4048
|
-
if (!isObj4(node))
|
|
4049
|
-
return;
|
|
4050
|
-
const types = typesOf(node);
|
|
4051
|
-
if (types.includes("ItemList")) {
|
|
4052
|
-
lists++;
|
|
4053
|
-
const elements = Array.isArray(node.itemListElement) ? node.itemListElement : [node.itemListElement];
|
|
4054
|
-
for (const el of elements) {
|
|
4055
|
-
if (!isObj4(el))
|
|
4056
|
-
continue;
|
|
4057
|
-
const item = isObj4(el.item) ? el.item : el;
|
|
4058
|
-
if (typesOf(item).includes("ItemList"))
|
|
4059
|
-
walk(item, depth + 1);
|
|
4060
|
-
else
|
|
4061
|
-
add(listingOf(item, pageUrl, el.url ?? (typeof el.item === "string" ? el.item : undefined)));
|
|
4062
|
-
}
|
|
4063
|
-
} else if (types.some((t) => PRODUCT_TYPES.has(t))) {
|
|
4064
|
-
add(listingOf(node, pageUrl, pageUrl));
|
|
4065
|
-
}
|
|
4066
|
-
if (node["@graph"] !== undefined)
|
|
4067
|
-
walk(node["@graph"], depth + 1);
|
|
4068
|
-
if (isObj4(node.mainEntity))
|
|
4069
|
-
walk(node.mainEntity, depth + 1);
|
|
4070
|
-
};
|
|
4071
|
-
for (const block of jsonLdBlocks(html))
|
|
4072
|
-
walk(block, 0);
|
|
4073
|
-
return { listings: out, lists };
|
|
4074
|
-
}
|
|
4075
|
-
var pageSource = {
|
|
4076
|
-
id: "page",
|
|
4077
|
-
async search(target, fetch2) {
|
|
4078
|
-
if (!fetch2)
|
|
4079
|
-
throw new SourceMiss("this bot reads no pages");
|
|
4080
|
-
let body;
|
|
4081
|
-
try {
|
|
4082
|
-
const response = await fetch2.get(target, { [NEVER_EBAY]: "1" });
|
|
4083
|
-
if (response.status !== 200)
|
|
4084
|
-
throw new SourceMiss(`HTTP ${response.status}`);
|
|
4085
|
-
body = response.body;
|
|
4086
|
-
} catch (err) {
|
|
4087
|
-
if (err instanceof SourceMiss)
|
|
4088
|
-
throw err;
|
|
4089
|
-
throw new SourceMiss(`the read failed: ${err instanceof Error ? err.message : String(err)}`, {
|
|
4090
|
-
cause: err
|
|
4091
|
-
});
|
|
4092
|
-
}
|
|
4093
|
-
const { listings, lists } = readListings(body, target);
|
|
4094
|
-
if (listings.length === 0 && lists === 0)
|
|
4095
|
-
throw new SourceMiss("no listings in the page's structured data");
|
|
4096
|
-
return listings;
|
|
4097
|
-
}
|
|
4098
|
-
};
|
|
4099
|
-
var nonNegative = (n) => n !== null && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
4100
|
-
function ebayItem(url) {
|
|
4101
|
-
if (url === null)
|
|
4102
|
-
return null;
|
|
4103
|
-
const u = new URL(url);
|
|
4104
|
-
const item = /^\/itm\/(?:[^/]+\/)?(\d{6,20})\/?$/.exec(u.pathname);
|
|
4105
|
-
return isEbayHost(u.hostname) && item ? `https://${u.hostname}/itm/${item[1]}` : url;
|
|
4106
|
-
}
|
|
4107
|
-
function submittedListing(raw) {
|
|
4108
|
-
const title = typeof raw.title === "string" ? clean(raw.title, TITLE_CHARS, true) : "";
|
|
4109
|
-
if (typeof raw.url !== "string" || !/^https?:\/\/[^/\\]/i.test(raw.url.trim()))
|
|
4110
|
-
return null;
|
|
4111
|
-
const url = ebayItem(absolute(raw.url, "https://invalid.example/"));
|
|
4112
|
-
if (!title || !url || url.startsWith("https://invalid.example/"))
|
|
4113
|
-
return null;
|
|
4114
|
-
const price = typeof raw.price === "number" && Number.isFinite(raw.price) && raw.price >= 0 ? raw.price : typeof raw.price === "string" ? nonNegative(parsePrice(raw.price)) : undefined;
|
|
4115
|
-
const currency = typeof raw.currency === "string" && /^[A-Za-z]{3}$/.test(raw.currency.trim()) ? raw.currency.trim().toUpperCase() : undefined;
|
|
4116
|
-
const condition = typeof raw.condition === "string" ? clean(raw.condition, CONDITION_CHARS, true) : "";
|
|
4117
|
-
const seller = typeof raw.seller === "string" ? clean(raw.seller, SELLER_CHARS, true) : "";
|
|
4118
|
-
return {
|
|
4119
|
-
id: url,
|
|
4120
|
-
title,
|
|
4121
|
-
url,
|
|
4122
|
-
...price !== undefined ? { price } : {},
|
|
4123
|
-
...currency ? { currency } : {},
|
|
4124
|
-
...condition ? { condition } : {},
|
|
4125
|
-
...seller ? { seller } : {}
|
|
4126
|
-
};
|
|
4127
|
-
}
|
|
4128
|
-
function inboxSource(read) {
|
|
4129
|
-
return {
|
|
4130
|
-
id: "inbox",
|
|
4131
|
-
async search(target) {
|
|
4132
|
-
let items;
|
|
4133
|
-
try {
|
|
4134
|
-
items = await read(target);
|
|
4135
|
-
} catch (err) {
|
|
4136
|
-
throw new SourceMiss("the inbox could not be read", { cause: err });
|
|
4137
|
-
}
|
|
4138
|
-
const byId = new Map;
|
|
4139
|
-
for (const raw of items) {
|
|
4140
|
-
const l = submittedListing(raw);
|
|
4141
|
-
if (!l)
|
|
4142
|
-
continue;
|
|
4143
|
-
byId.delete(l.id);
|
|
4144
|
-
byId.set(l.id, l);
|
|
4145
|
-
}
|
|
4146
|
-
return [...byId.values()].slice(-MAX_LISTINGS);
|
|
4147
|
-
}
|
|
4148
|
-
};
|
|
4149
|
-
}
|
|
4259
|
+
capabilities: ["notify"],
|
|
4260
|
+
schedule: ["calendar"],
|
|
4261
|
+
intake: {
|
|
4262
|
+
options: [
|
|
4263
|
+
{
|
|
4264
|
+
name: "interests",
|
|
4265
|
+
description: "What they are into, separated by commas",
|
|
4266
|
+
required: true,
|
|
4267
|
+
kind: "string"
|
|
4268
|
+
},
|
|
4269
|
+
{
|
|
4270
|
+
name: "lens",
|
|
4271
|
+
description: "The gift lens: general, birthday, anniversary or christmas",
|
|
4272
|
+
required: false,
|
|
4273
|
+
kind: "string"
|
|
4274
|
+
},
|
|
4275
|
+
{
|
|
4276
|
+
name: "for",
|
|
4277
|
+
description: "Who the ideas are for, if not you",
|
|
4278
|
+
required: false,
|
|
4279
|
+
kind: "string"
|
|
4280
|
+
},
|
|
4281
|
+
{ name: "notes", description: "Anything else to weigh", required: false, kind: "string" },
|
|
4282
|
+
{ name: "every", description: "Days between runs", required: false, kind: "integer" }
|
|
4283
|
+
]
|
|
4284
|
+
},
|
|
4285
|
+
async prepare(ctx) {
|
|
4286
|
+
return scoutJob(ctx.config, scoutState(ctx.state));
|
|
4287
|
+
},
|
|
4288
|
+
async finish(ctx, result) {
|
|
4289
|
+
const stored = scoutState(ctx.state);
|
|
4290
|
+
const state = ctx.occurrence.dedupeKey.startsWith(FOLLOW_UP_PREFIX) ? stored : { ...stored, failures: 0 };
|
|
4291
|
+
if (result.kind !== "success")
|
|
4292
|
+
return failed2(ctx, state, result.kind, result.detail);
|
|
4293
|
+
return afterRun(ctx, state, result.structuredOutput);
|
|
4294
|
+
}
|
|
4295
|
+
});
|
|
4150
4296
|
|
|
4151
4297
|
// node_modules/@rackbops/docket-types/dist/want-bgg.js
|
|
4152
4298
|
var BGG_HOST = "boardgamegeek.com";
|
|
@@ -4307,11 +4453,8 @@ var SHOPIFY_COUNTRY = "US";
|
|
|
4307
4453
|
var SHOPIFY_CURRENCY = "USD";
|
|
4308
4454
|
var SHOPIFY_ATTRIBUTION = "via Shopify";
|
|
4309
4455
|
var MAX_VARIANTS = 20;
|
|
4310
|
-
var TITLE_CHARS2 = 150;
|
|
4311
|
-
var SELLER_CHARS2 = 80;
|
|
4312
4456
|
function shopifyRequest(s) {
|
|
4313
|
-
const
|
|
4314
|
-
const currency = /^[A-Z]{3}$/.test(asked) ? asked : SHOPIFY_CURRENCY;
|
|
4457
|
+
const currency = currencyCode(s.currency) ?? SHOPIFY_CURRENCY;
|
|
4315
4458
|
const filters = { available: true };
|
|
4316
4459
|
if (s.maxPrice !== undefined && Number.isFinite(s.maxPrice) && s.maxPrice >= 0)
|
|
4317
4460
|
filters.price = { max: Math.floor(s.maxPrice * minorUnits(currency) + 0.000001) };
|
|
@@ -4327,7 +4470,6 @@ function shopifyRequest(s) {
|
|
|
4327
4470
|
}
|
|
4328
4471
|
};
|
|
4329
4472
|
}
|
|
4330
|
-
var isObj5 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4331
4473
|
var QUERY_CHARS = 200;
|
|
4332
4474
|
function shopifyQuery(query) {
|
|
4333
4475
|
const line = clean(query, 1e4, true);
|
|
@@ -4364,14 +4506,8 @@ function listingUrl(raw) {
|
|
|
4364
4506
|
u.searchParams.delete(key);
|
|
4365
4507
|
return safeUrl(u.toString());
|
|
4366
4508
|
}
|
|
4367
|
-
var ZERO_DECIMAL = new Set("BIF CLP DJF GNF ISK JPY KMF KRW PYG RWF UGX UYI VND VUV XAF XOF XPF".split(" "));
|
|
4368
|
-
var THREE_DECIMAL = new Set("BHD IQD JOD KWD LYD OMR TND".split(" "));
|
|
4369
|
-
function minorUnits(currency) {
|
|
4370
|
-
const c = currency.toUpperCase();
|
|
4371
|
-
return ZERO_DECIMAL.has(c) ? 1 : THREE_DECIMAL.has(c) ? 1000 : 100;
|
|
4372
|
-
}
|
|
4373
4509
|
function priceOf(v) {
|
|
4374
|
-
if (!
|
|
4510
|
+
if (!isObj3(v))
|
|
4375
4511
|
return null;
|
|
4376
4512
|
const amount = v.amount;
|
|
4377
4513
|
const currency = typeof v.currency === "string" ? v.currency.trim().toUpperCase() : "";
|
|
@@ -4389,17 +4525,17 @@ function isNew(variant) {
|
|
|
4389
4525
|
return all.every((x) => typeof x === "string" && x.trim().toLowerCase() === "new");
|
|
4390
4526
|
}
|
|
4391
4527
|
function parseShopifyAnswer(body, query) {
|
|
4392
|
-
const result =
|
|
4528
|
+
const result = isObj3(body) && isObj3(body.data) ? body.data.alexandria : undefined;
|
|
4393
4529
|
const first = Array.isArray(result) ? result[0] : undefined;
|
|
4394
|
-
if (!
|
|
4530
|
+
if (!isObj3(first))
|
|
4395
4531
|
throw new SourceMiss("Firecrawl's answer had no search result");
|
|
4396
4532
|
if (first.error !== undefined && first.error !== null) {
|
|
4397
4533
|
const e = first.error;
|
|
4398
|
-
const why =
|
|
4534
|
+
const why = isObj3(e) && typeof e.code === "string" ? clean(e.code, 60, true) : "an error";
|
|
4399
4535
|
throw new SourceMiss(`Shopify's search failed (${why})`);
|
|
4400
4536
|
}
|
|
4401
4537
|
const data = first.data;
|
|
4402
|
-
if (!
|
|
4538
|
+
if (!isObj3(data) || !Array.isArray(data.products))
|
|
4403
4539
|
throw new SourceMiss("Shopify's search answer had no products");
|
|
4404
4540
|
const words = searchWords(query);
|
|
4405
4541
|
const out = [];
|
|
@@ -4407,32 +4543,32 @@ function parseShopifyAnswer(body, query) {
|
|
|
4407
4543
|
for (const product of data.products.slice(0, SHOPIFY_LIMIT)) {
|
|
4408
4544
|
if (out.length >= MAX_LISTINGS)
|
|
4409
4545
|
break;
|
|
4410
|
-
if (!
|
|
4546
|
+
if (!isObj3(product) || typeof product.title !== "string")
|
|
4411
4547
|
continue;
|
|
4412
4548
|
if (!matchesWords(product.title.slice(0, 2000), words))
|
|
4413
4549
|
continue;
|
|
4414
|
-
const title = clean(product.title,
|
|
4550
|
+
const title = clean(product.title, TITLE_CHARS, true);
|
|
4415
4551
|
if (!title)
|
|
4416
4552
|
continue;
|
|
4417
4553
|
const variants = Array.isArray(product.variants) ? product.variants.slice(0, MAX_VARIANTS) : [];
|
|
4418
4554
|
for (const v of variants) {
|
|
4419
4555
|
if (out.length >= MAX_LISTINGS)
|
|
4420
4556
|
break;
|
|
4421
|
-
if (!
|
|
4557
|
+
if (!isObj3(v) || typeof v.id !== "string" || v.id.length > 300 || !isNew(v))
|
|
4422
4558
|
continue;
|
|
4423
|
-
if (!
|
|
4559
|
+
if (!isObj3(v.availability) || v.availability.available !== true)
|
|
4424
4560
|
continue;
|
|
4425
4561
|
const url = listingUrl(v.url);
|
|
4426
4562
|
const id = `shopify:${v.id}`;
|
|
4427
4563
|
if (!url || seen.has(id))
|
|
4428
4564
|
continue;
|
|
4429
4565
|
seen.add(id);
|
|
4430
|
-
const own = typeof v.title === "string" ? clean(v.title,
|
|
4566
|
+
const own = typeof v.title === "string" ? clean(v.title, TITLE_CHARS, true) : "";
|
|
4431
4567
|
const named = own !== "" && own.toLowerCase() !== "default title" && !title.toLowerCase().includes(own.toLowerCase());
|
|
4432
|
-
const name = named ? clean(`${title} (${own})`,
|
|
4568
|
+
const name = named ? clean(`${title} (${own})`, TITLE_CHARS, true) : title;
|
|
4433
4569
|
const price = priceOf(v.price);
|
|
4434
|
-
const seller =
|
|
4435
|
-
const sold = clean(seller,
|
|
4570
|
+
const seller = isObj3(v.seller) && typeof v.seller.name === "string" ? v.seller.name : "";
|
|
4571
|
+
const sold = clean(seller, SELLER_CHARS, true);
|
|
4436
4572
|
out.push({
|
|
4437
4573
|
id,
|
|
4438
4574
|
title: name,
|
|
@@ -4458,12 +4594,9 @@ function shopifySource(scrape) {
|
|
|
4458
4594
|
} catch (err) {
|
|
4459
4595
|
throw new SourceMiss(`Shopify could not be searched: ${clean(err instanceof Error ? err.message : String(err), 200, true)}`, { cause: err });
|
|
4460
4596
|
}
|
|
4461
|
-
|
|
4462
|
-
|
|
4463
|
-
|
|
4464
|
-
throw new SourceMiss(`Firecrawl is busy (HTTP ${status})`);
|
|
4465
|
-
if (status !== 200)
|
|
4466
|
-
throw new SourceMiss(`Firecrawl answered HTTP ${status}`);
|
|
4597
|
+
checkSearchStatus(status, "Firecrawl", "Firecrawl refused the key", {
|
|
4598
|
+
402: "the Firecrawl account is out of credits"
|
|
4599
|
+
});
|
|
4467
4600
|
return parseShopifyAnswer(body, target);
|
|
4468
4601
|
}
|
|
4469
4602
|
};
|
|
@@ -4475,12 +4608,8 @@ var EBAY_MARKETPLACE = "EBAY_US";
|
|
|
4475
4608
|
var EBAY_CURRENCY = "USD";
|
|
4476
4609
|
var EBAY_LIMIT = 50;
|
|
4477
4610
|
var EBAY_ATTRIBUTION = "via eBay";
|
|
4478
|
-
var TITLE_CHARS3 = 150;
|
|
4479
|
-
var CONDITION_CHARS2 = 40;
|
|
4480
|
-
var SELLER_CHARS3 = 80;
|
|
4481
4611
|
function ebayRequest(s) {
|
|
4482
|
-
const
|
|
4483
|
-
const currency = asked === "" ? EBAY_CURRENCY : asked;
|
|
4612
|
+
const currency = currencyCode(s.currency) ?? EBAY_CURRENCY;
|
|
4484
4613
|
const params = {
|
|
4485
4614
|
q: shopifyQuery(s.query),
|
|
4486
4615
|
limit: String(EBAY_LIMIT),
|
|
@@ -4492,9 +4621,8 @@ function ebayRequest(s) {
|
|
|
4492
4621
|
}
|
|
4493
4622
|
return { path: EBAY_SEARCH_PATH, params, marketplace: EBAY_MARKETPLACE };
|
|
4494
4623
|
}
|
|
4495
|
-
var isObj6 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4496
4624
|
function amountOf(v) {
|
|
4497
|
-
if (!
|
|
4625
|
+
if (!isObj3(v))
|
|
4498
4626
|
return null;
|
|
4499
4627
|
const raw = typeof v.value === "string" || typeof v.value === "number" ? String(v.value) : "";
|
|
4500
4628
|
const currency = typeof v.currency === "string" ? v.currency.trim().toUpperCase() : "";
|
|
@@ -4517,7 +4645,7 @@ function itemUrl(raw) {
|
|
|
4517
4645
|
return item ? safeUrl(`https://${u.hostname.toLowerCase()}/itm/${item[1]}`) : null;
|
|
4518
4646
|
}
|
|
4519
4647
|
function sellerOf(v) {
|
|
4520
|
-
if (!
|
|
4648
|
+
if (!isObj3(v) || typeof v.username !== "string")
|
|
4521
4649
|
return "";
|
|
4522
4650
|
const name = clean(v.username, 40, true);
|
|
4523
4651
|
if (!name)
|
|
@@ -4528,10 +4656,10 @@ function sellerOf(v) {
|
|
|
4528
4656
|
parts.push(`${pct}% positive`);
|
|
4529
4657
|
if (typeof v.feedbackScore === "number" && Number.isInteger(v.feedbackScore))
|
|
4530
4658
|
parts.push(`${v.feedbackScore} feedback`);
|
|
4531
|
-
return clean(parts.join(", "),
|
|
4659
|
+
return clean(parts.join(", "), SELLER_CHARS, true);
|
|
4532
4660
|
}
|
|
4533
4661
|
function parseEbayAnswer(body, query) {
|
|
4534
|
-
if (!
|
|
4662
|
+
if (!isObj3(body) || body.itemSummaries === undefined && typeof body.total !== "number")
|
|
4535
4663
|
throw new SourceMiss("eBay's answer was not a search result");
|
|
4536
4664
|
const items = Array.isArray(body.itemSummaries) ? body.itemSummaries : [];
|
|
4537
4665
|
const words = searchWords(query);
|
|
@@ -4540,11 +4668,11 @@ function parseEbayAnswer(body, query) {
|
|
|
4540
4668
|
for (const item of items.slice(0, EBAY_LIMIT * 4)) {
|
|
4541
4669
|
if (out.length >= MAX_LISTINGS)
|
|
4542
4670
|
break;
|
|
4543
|
-
if (!
|
|
4671
|
+
if (!isObj3(item) || typeof item.title !== "string" || typeof item.itemId !== "string")
|
|
4544
4672
|
continue;
|
|
4545
4673
|
if (!matchesWords(item.title.slice(0, 2000), words))
|
|
4546
4674
|
continue;
|
|
4547
|
-
const title = clean(item.title,
|
|
4675
|
+
const title = clean(item.title, TITLE_CHARS, true);
|
|
4548
4676
|
const url = itemUrl(item.itemWebUrl);
|
|
4549
4677
|
if (!title || !url || item.itemId.length > 120)
|
|
4550
4678
|
continue;
|
|
@@ -4558,7 +4686,7 @@ function parseEbayAnswer(body, query) {
|
|
|
4558
4686
|
const price = amountOf(auction && !buyNow ? item.currentBidPrice ?? item.price : item.price);
|
|
4559
4687
|
const state = typeof item.condition === "string" ? clean(item.condition, 30, true) : "";
|
|
4560
4688
|
const kind = auction ? [buyNow ? "auction, Buy It Now" : "auction"] : [];
|
|
4561
|
-
const condition = clean([state, ...kind].filter(Boolean).join(", "),
|
|
4689
|
+
const condition = clean([state, ...kind].filter(Boolean).join(", "), CONDITION_CHARS, true);
|
|
4562
4690
|
const seller = sellerOf(item.seller);
|
|
4563
4691
|
out.push({
|
|
4564
4692
|
id: `ebay:${number}`,
|
|
@@ -4584,17 +4712,49 @@ function ebaySource(browse) {
|
|
|
4584
4712
|
} catch (err) {
|
|
4585
4713
|
throw new SourceMiss(`eBay could not be searched: ${clean(err instanceof Error ? err.message : String(err), 200, true)}`, { cause: err });
|
|
4586
4714
|
}
|
|
4587
|
-
|
|
4588
|
-
throw new SourceUnavailableError("eBay refused the app's keys");
|
|
4589
|
-
if (status === 429 || status >= 500)
|
|
4590
|
-
throw new SourceMiss(`eBay is busy (HTTP ${status})`);
|
|
4591
|
-
if (status !== 200)
|
|
4592
|
-
throw new SourceMiss(`eBay answered HTTP ${status}`);
|
|
4715
|
+
checkSearchStatus(status, "eBay", "eBay refused the app's keys");
|
|
4593
4716
|
return parseEbayAnswer(body, target);
|
|
4594
4717
|
}
|
|
4595
4718
|
};
|
|
4596
4719
|
}
|
|
4597
4720
|
|
|
4721
|
+
// node_modules/@rackbops/docket-types/dist/want-text.js
|
|
4722
|
+
var SOURCE_TEXT = {
|
|
4723
|
+
page: {
|
|
4724
|
+
attribution: null,
|
|
4725
|
+
where: null,
|
|
4726
|
+
missing: "such source",
|
|
4727
|
+
judgeWhere: "a shop page they gave"
|
|
4728
|
+
},
|
|
4729
|
+
bgg: {
|
|
4730
|
+
attribution: BGG_ATTRIBUTION,
|
|
4731
|
+
where: "BoardGameGeek",
|
|
4732
|
+
missing: "BoardGameGeek access",
|
|
4733
|
+
judgeWhere: "BoardGameGeek's marketplace"
|
|
4734
|
+
},
|
|
4735
|
+
inbox: {
|
|
4736
|
+
attribution: null,
|
|
4737
|
+
where: "what was sent in",
|
|
4738
|
+
missing: "such source",
|
|
4739
|
+
judgeWhere: "alert emails and searches sent in for them (from eBay, BoardGameGeek or a shop)"
|
|
4740
|
+
},
|
|
4741
|
+
shopify: {
|
|
4742
|
+
attribution: SHOPIFY_ATTRIBUTION,
|
|
4743
|
+
where: "Shopify's store search",
|
|
4744
|
+
missing: "Shopify search (no Firecrawl key)",
|
|
4745
|
+
judgeWhere: "a search of Shopify stores' catalogs (new copies in stock)"
|
|
4746
|
+
},
|
|
4747
|
+
ebay: {
|
|
4748
|
+
attribution: EBAY_ATTRIBUTION,
|
|
4749
|
+
where: "eBay's search",
|
|
4750
|
+
missing: "eBay search (no eBay keys)",
|
|
4751
|
+
judgeWhere: "a search of eBay's listings, through eBay's own API"
|
|
4752
|
+
}
|
|
4753
|
+
};
|
|
4754
|
+
function sourceText(source) {
|
|
4755
|
+
return Object.hasOwn(SOURCE_TEXT, source) ? SOURCE_TEXT[source] : SOURCE_TEXT.page;
|
|
4756
|
+
}
|
|
4757
|
+
|
|
4598
4758
|
// node_modules/@rackbops/docket-types/dist/wantlist.js
|
|
4599
4759
|
var MAX_REPORTED = 500;
|
|
4600
4760
|
var SHOWN_IN_DM = 5;
|
|
@@ -4624,13 +4784,53 @@ function wantState(value) {
|
|
|
4624
4784
|
};
|
|
4625
4785
|
}
|
|
4626
4786
|
function withinLimits(l, config) {
|
|
4627
|
-
|
|
4628
|
-
|
|
4787
|
+
if (l.price !== undefined && !(l.price >= 0))
|
|
4788
|
+
return false;
|
|
4789
|
+
const want = currencyCode(config.currency);
|
|
4790
|
+
if (want && l.currency && currencyCode(l.currency) !== want)
|
|
4629
4791
|
return false;
|
|
4630
4792
|
if (config.maxPrice === undefined)
|
|
4631
4793
|
return true;
|
|
4632
4794
|
return l.price !== undefined && l.price <= config.maxPrice + 0.000000001;
|
|
4633
4795
|
}
|
|
4796
|
+
var MAX_EXCLUDE = 20;
|
|
4797
|
+
var EXCLUDE_CHARS = 80;
|
|
4798
|
+
function parseExclude(text) {
|
|
4799
|
+
const out = [];
|
|
4800
|
+
const seen = new Set;
|
|
4801
|
+
for (const raw of text.split(/[,\n]/)) {
|
|
4802
|
+
const entry = raw.trim().slice(0, EXCLUDE_CHARS).trim();
|
|
4803
|
+
const key = wordsOf(entry).join(" ");
|
|
4804
|
+
if (!key || seen.has(key))
|
|
4805
|
+
continue;
|
|
4806
|
+
seen.add(key);
|
|
4807
|
+
out.push(entry);
|
|
4808
|
+
if (out.length === MAX_EXCLUDE)
|
|
4809
|
+
break;
|
|
4810
|
+
}
|
|
4811
|
+
return out;
|
|
4812
|
+
}
|
|
4813
|
+
function isExcluded(title, value) {
|
|
4814
|
+
const exclude = typeof value === "string" ? parseExclude(value) : value;
|
|
4815
|
+
if (!Array.isArray(exclude) || exclude.length === 0)
|
|
4816
|
+
return false;
|
|
4817
|
+
const have = wordsOf(title);
|
|
4818
|
+
return exclude.slice(0, MAX_EXCLUDE).some((entry) => {
|
|
4819
|
+
if (typeof entry !== "string")
|
|
4820
|
+
return false;
|
|
4821
|
+
const want = wordsOf(entry.slice(0, EXCLUDE_CHARS));
|
|
4822
|
+
if (want.length === 0)
|
|
4823
|
+
return false;
|
|
4824
|
+
for (let i = 0;i + want.length <= have.length; i++) {
|
|
4825
|
+
if (want.every((w, j) => have[i + j] === w))
|
|
4826
|
+
return true;
|
|
4827
|
+
}
|
|
4828
|
+
return false;
|
|
4829
|
+
});
|
|
4830
|
+
}
|
|
4831
|
+
function wanted(l, config) {
|
|
4832
|
+
return withinLimits(l, config) && !isExcluded(l.title, config.exclude);
|
|
4833
|
+
}
|
|
4634
4834
|
function priceText(l, config) {
|
|
4635
4835
|
return l.price === undefined ? "price not shown" : money(l.price, l.currency ?? config.currency ?? "");
|
|
4636
4836
|
}
|
|
@@ -4641,7 +4841,8 @@ function listingLine(l, config) {
|
|
|
4641
4841
|
...l.condition ? [inert(l.condition)] : [],
|
|
4642
4842
|
...l.seller ? [`sold by ${inert(l.seller)}`] : []
|
|
4643
4843
|
];
|
|
4644
|
-
const
|
|
4844
|
+
const attribution = sourceText(config.source).attribution;
|
|
4845
|
+
const via = attribution ? ` (${attribution})` : "";
|
|
4645
4846
|
return `- ${parts.join(" -- ")} <${l.url}>${via}`;
|
|
4646
4847
|
}
|
|
4647
4848
|
function renderWant(title, taskId, fresh, config) {
|
|
@@ -4665,7 +4866,7 @@ function miss2(ctx, state, err) {
|
|
|
4665
4866
|
summary: `nothing read: ${reason}`
|
|
4666
4867
|
};
|
|
4667
4868
|
if (tell) {
|
|
4668
|
-
const where = ctx.config.source
|
|
4869
|
+
const where = sourceText(ctx.config.source).where ?? ctx.config.target;
|
|
4669
4870
|
outcome.notify = {
|
|
4670
4871
|
text: clip(`${ctx.task.title}: I could read no listings from ${where} ${misses === 1 ? "just now" : `${misses} times in a row`} (${reason}). ` + "I keep looking; press Done to stop."),
|
|
4671
4872
|
actions: ["done"]
|
|
@@ -4687,7 +4888,14 @@ async function pollWant(ctx, sources, stored) {
|
|
|
4687
4888
|
if (!source) {
|
|
4688
4889
|
return {
|
|
4689
4890
|
kind: "done",
|
|
4690
|
-
outcome: miss2(ctx, state, new SourceUnavailableError(`this bot has no ${ctx.config.source
|
|
4891
|
+
outcome: miss2(ctx, state, new SourceUnavailableError(`this bot has no ${sourceText(ctx.config.source).missing}`))
|
|
4892
|
+
};
|
|
4893
|
+
}
|
|
4894
|
+
const currency = currencyCode(ctx.config.currency);
|
|
4895
|
+
if (ctx.config.source === "ebay" && currency && currency !== EBAY_CURRENCY) {
|
|
4896
|
+
return {
|
|
4897
|
+
kind: "done",
|
|
4898
|
+
outcome: miss2(ctx, state, new SourceUnavailableError(`eBay US lists prices in ${EBAY_CURRENCY} only, so a watch in ${currency} never matches; ` + `set the currency to ${EBAY_CURRENCY} or leave it out`))
|
|
4691
4899
|
};
|
|
4692
4900
|
}
|
|
4693
4901
|
let listings;
|
|
@@ -4702,9 +4910,11 @@ async function pollWant(ctx, sources, stored) {
|
|
|
4702
4910
|
throw err;
|
|
4703
4911
|
}
|
|
4704
4912
|
const told = new Set(state.reported);
|
|
4705
|
-
const
|
|
4913
|
+
const limited = listings.filter((l) => withinLimits(l, ctx.config));
|
|
4914
|
+
const within = limited.filter((l) => !isExcluded(l.title, ctx.config.exclude));
|
|
4706
4915
|
const fresh = within.filter((l) => !told.has(listingKey(l.id))).slice(0, MAX_NEW_PER_RUN);
|
|
4707
|
-
const
|
|
4916
|
+
const dropped = limited.length - within.length;
|
|
4917
|
+
const summary = `${listings.length} listed, ${limited.length} within limits, ` + `${dropped > 0 ? `${dropped} excluded, ` : ""}${fresh.length} new`;
|
|
4708
4918
|
const read = { ...state, misses: 0, warned: false };
|
|
4709
4919
|
if (fresh.length === 0)
|
|
4710
4920
|
return { kind: "done", outcome: { state: read, summary } };
|
|
@@ -4741,6 +4951,12 @@ function wantlistType(sources) {
|
|
|
4741
4951
|
description: "Only listings in this currency",
|
|
4742
4952
|
required: false,
|
|
4743
4953
|
kind: "string"
|
|
4954
|
+
},
|
|
4955
|
+
{
|
|
4956
|
+
name: "exclude",
|
|
4957
|
+
description: "Leave out listings whose title has any of these words or phrases, separated by commas",
|
|
4958
|
+
required: false,
|
|
4959
|
+
kind: "string"
|
|
4744
4960
|
}
|
|
4745
4961
|
]
|
|
4746
4962
|
},
|
|
@@ -4778,7 +4994,7 @@ var JUDGE_MAX_BUDGET_USD = 0.5;
|
|
|
4778
4994
|
var JUDGE_TIMEOUT_MS = 300000;
|
|
4779
4995
|
var RETRY_GRACE_MS = 30 * 60000;
|
|
4780
4996
|
var WHY_CHARS = 200;
|
|
4781
|
-
var
|
|
4997
|
+
var SELLER_CHARS2 = 200;
|
|
4782
4998
|
var PENDING_TITLE = 150;
|
|
4783
4999
|
var PENDING_TEXT = 80;
|
|
4784
5000
|
var JUDGE_SCHEMA = {
|
|
@@ -4801,10 +5017,9 @@ var JUDGE_SCHEMA = {
|
|
|
4801
5017
|
},
|
|
4802
5018
|
required: ["verdicts"]
|
|
4803
5019
|
};
|
|
4804
|
-
var isObj7 = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4805
5020
|
var str = (v, max) => typeof v === "string" ? clean(v, max, true) : "";
|
|
4806
5021
|
function pendingListing(v) {
|
|
4807
|
-
if (!
|
|
5022
|
+
if (!isObj3(v) || typeof v.id !== "string")
|
|
4808
5023
|
return null;
|
|
4809
5024
|
const url = safeUrl(v.url);
|
|
4810
5025
|
const title = str(v.title, PENDING_TITLE);
|
|
@@ -4825,7 +5040,7 @@ function pendingListing(v) {
|
|
|
4825
5040
|
}
|
|
4826
5041
|
function judgeState(value) {
|
|
4827
5042
|
const base = wantState(value);
|
|
4828
|
-
const s =
|
|
5043
|
+
const s = isObj3(value) ? value : {};
|
|
4829
5044
|
const pending = (Array.isArray(s.pending) ? s.pending : []).map(pendingListing).filter((l) => l !== null).slice(0, MAX_NEW_PER_RUN);
|
|
4830
5045
|
const failures = typeof s.failures === "number" && s.failures >= 0 ? s.failures : 0;
|
|
4831
5046
|
const retryAt = typeof s.retryAt === "string" && Number.isFinite(Date.parse(s.retryAt)) ? s.retryAt : undefined;
|
|
@@ -4874,13 +5089,14 @@ function closedSource(config) {
|
|
|
4874
5089
|
return config.source === "inbox" || config.source === "shopify" || config.source === "ebay";
|
|
4875
5090
|
}
|
|
4876
5091
|
function limitsText(config) {
|
|
5092
|
+
const currency = currencyCode(config.currency) ?? "";
|
|
4877
5093
|
if (config.maxPrice !== undefined)
|
|
4878
|
-
return `at or under ${money(config.maxPrice,
|
|
4879
|
-
return
|
|
5094
|
+
return `at or under ${money(config.maxPrice, currency)}`;
|
|
5095
|
+
return currency ? `any price, in ${currency}` : "any price";
|
|
4880
5096
|
}
|
|
4881
5097
|
function judgeJob(title, config, pending) {
|
|
4882
5098
|
const closed = closedSource(config);
|
|
4883
|
-
const where = config.source
|
|
5099
|
+
const where = sourceText(config.source).judgeWhere;
|
|
4884
5100
|
const lines = pending.map((l, i) => {
|
|
4885
5101
|
const parts = [
|
|
4886
5102
|
inert(l.title),
|
|
@@ -4926,18 +5142,18 @@ function judgeJob(title, config, pending) {
|
|
|
4926
5142
|
};
|
|
4927
5143
|
}
|
|
4928
5144
|
function parseVerdicts(value, count) {
|
|
4929
|
-
if (!
|
|
5145
|
+
if (!isObj3(value) || !Array.isArray(value.verdicts))
|
|
4930
5146
|
return null;
|
|
4931
5147
|
const out = new Map;
|
|
4932
5148
|
for (const v of value.verdicts) {
|
|
4933
|
-
if (!
|
|
5149
|
+
if (!isObj3(v) || typeof v.n !== "number" || !Number.isInteger(v.n) || v.n < 1 || v.n > count || out.has(v.n))
|
|
4934
5150
|
continue;
|
|
4935
5151
|
if (typeof v.fit !== "string" || !FITS.includes(v.fit))
|
|
4936
5152
|
continue;
|
|
4937
5153
|
out.set(v.n, {
|
|
4938
5154
|
fit: v.fit,
|
|
4939
5155
|
why: inert(str(v.why, WHY_CHARS)),
|
|
4940
|
-
seller: inert(str(v.seller,
|
|
5156
|
+
seller: inert(str(v.seller, SELLER_CHARS2))
|
|
4941
5157
|
});
|
|
4942
5158
|
}
|
|
4943
5159
|
return out;
|
|
@@ -4984,13 +5200,6 @@ function settled(state, told) {
|
|
|
4984
5200
|
failures: 0
|
|
4985
5201
|
};
|
|
4986
5202
|
}
|
|
4987
|
-
var RETRIED3 = new Set([
|
|
4988
|
-
"schema_miss",
|
|
4989
|
-
"malformed",
|
|
4990
|
-
"timeout",
|
|
4991
|
-
"error",
|
|
4992
|
-
"auth_failed"
|
|
4993
|
-
]);
|
|
4994
5203
|
var SAID3 = {
|
|
4995
5204
|
auth_failed: "the model runner could not sign in",
|
|
4996
5205
|
turn_cap: "the check needed more steps than it is allowed",
|
|
@@ -5003,13 +5212,12 @@ var SAID3 = {
|
|
|
5003
5212
|
function judgeFailed(ctx, state, kind, detail) {
|
|
5004
5213
|
const failures = state.failures + 1;
|
|
5005
5214
|
const summary = clean(`judge ${kind} (try ${failures}): ${detail}`, 300, true);
|
|
5006
|
-
|
|
5007
|
-
|
|
5008
|
-
|
|
5009
|
-
return { state: { ...state, failures, retryAt: at }, followUp: wait > 0 ? { at } : {}, summary };
|
|
5215
|
+
const retry = retryOutcome(kind, failures, ctx.now);
|
|
5216
|
+
if (retry) {
|
|
5217
|
+
return { state: { ...state, failures, retryAt: retry.at }, followUp: retry.followUp, summary };
|
|
5010
5218
|
}
|
|
5011
5219
|
const reason = SAID3[kind] ?? "the check failed";
|
|
5012
|
-
const within = state.pending.filter((l) =>
|
|
5220
|
+
const within = state.pending.filter((l) => wanted(l, ctx.config));
|
|
5013
5221
|
const kept = { ...state, pending: within };
|
|
5014
5222
|
return {
|
|
5015
5223
|
state: settled(kept, within.length),
|
|
@@ -5026,7 +5234,7 @@ function judgeFailed(ctx, state, kind, detail) {
|
|
|
5026
5234
|
};
|
|
5027
5235
|
}
|
|
5028
5236
|
function judged(ctx, stored, verdicts) {
|
|
5029
|
-
const all = stored.pending.map((l, i) => ({ l, v: verdicts.get(i + 1) ?? { fit: "maybe", why: "", seller: "" } })).filter(({ l }) =>
|
|
5237
|
+
const all = stored.pending.map((l, i) => ({ l, v: verdicts.get(i + 1) ?? { fit: "maybe", why: "", seller: "" } })).filter(({ l }) => wanted(l, ctx.config));
|
|
5030
5238
|
const state = { ...stored, pending: all.map((x) => x.l) };
|
|
5031
5239
|
const rank = { match: 0, maybe: 1, no: 2 };
|
|
5032
5240
|
const shown = all.filter((x) => x.v.fit !== "no").sort((a, b) => rank[a.v.fit] - rank[b.v.fit]);
|
|
@@ -5087,12 +5295,18 @@ function wantjudgeType(sources, o = {}) {
|
|
|
5087
5295
|
description: "Only listings in this currency",
|
|
5088
5296
|
required: false,
|
|
5089
5297
|
kind: "string"
|
|
5298
|
+
},
|
|
5299
|
+
{
|
|
5300
|
+
name: "exclude",
|
|
5301
|
+
description: "Leave out listings whose title has any of these words or phrases, separated by commas",
|
|
5302
|
+
required: false,
|
|
5303
|
+
kind: "string"
|
|
5090
5304
|
}
|
|
5091
5305
|
]
|
|
5092
5306
|
},
|
|
5093
5307
|
async prepare(ctx) {
|
|
5094
5308
|
const state = judgeState(ctx.state);
|
|
5095
|
-
const followUp = ctx.occurrence.dedupeKey.startsWith(
|
|
5309
|
+
const followUp = ctx.occurrence.dedupeKey.startsWith(FOLLOW_UP_PREFIX);
|
|
5096
5310
|
if (state.pending.length > 0) {
|
|
5097
5311
|
const overdue = state.retryAt !== undefined && ctx.now.getTime() > Date.parse(state.retryAt) + RETRY_GRACE_MS;
|
|
5098
5312
|
if (state.failures > 0 && !followUp && !overdue)
|
|
@@ -5244,7 +5458,7 @@ function intakeJob(ctx) {
|
|
|
5244
5458
|
timeoutMs: INTAKE_TIMEOUT_MS
|
|
5245
5459
|
};
|
|
5246
5460
|
}
|
|
5247
|
-
function
|
|
5461
|
+
function isObj5(v) {
|
|
5248
5462
|
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
5249
5463
|
}
|
|
5250
5464
|
function checkDraft(draft, types, tasks = []) {
|
|
@@ -5301,7 +5515,7 @@ function checkDraft(draft, types, tasks = []) {
|
|
|
5301
5515
|
return null;
|
|
5302
5516
|
}
|
|
5303
5517
|
function readDraft(value) {
|
|
5304
|
-
if (!
|
|
5518
|
+
if (!isObj5(value) || !isObj5(value.fields) || typeof value.type !== "string")
|
|
5305
5519
|
return null;
|
|
5306
5520
|
if (value.action !== "create" && value.action !== "edit")
|
|
5307
5521
|
return null;
|
|
@@ -5326,7 +5540,7 @@ function readIntake(result, types, tasks = []) {
|
|
|
5326
5540
|
if (result.kind !== "success")
|
|
5327
5541
|
return { ok: false, retry: false, problem: result.kind };
|
|
5328
5542
|
const v = result.structuredOutput;
|
|
5329
|
-
if (!
|
|
5543
|
+
if (!isObj5(v) || typeof v.reply !== "string" || !INTAKE_STATES.includes(v.state)) {
|
|
5330
5544
|
return {
|
|
5331
5545
|
ok: false,
|
|
5332
5546
|
retry: true,
|
|
@@ -6214,11 +6428,21 @@ class SqliteStore {
|
|
|
6214
6428
|
params.push(expected);
|
|
6215
6429
|
return this.run(`UPDATE occurrences SET ${sets.join(", ")} WHERE seq = ?${guard}`, ...params).changes > 0;
|
|
6216
6430
|
}
|
|
6217
|
-
async deleteOccurrence(id) {
|
|
6431
|
+
async deleteOccurrence(id, guard) {
|
|
6218
6432
|
const seq = seqOf("o", id);
|
|
6219
6433
|
if (seq === null)
|
|
6220
6434
|
return false;
|
|
6221
|
-
|
|
6435
|
+
if (guard === undefined)
|
|
6436
|
+
return this.run("DELETE FROM occurrences WHERE seq = ? AND status = 'queued'", seq).changes > 0;
|
|
6437
|
+
const last = guard.lastEventId === null ? null : seqOf("e", guard.lastEventId);
|
|
6438
|
+
if (guard.lastEventId !== null && last === null)
|
|
6439
|
+
return false;
|
|
6440
|
+
if (last === null) {
|
|
6441
|
+
return this.run("DELETE FROM occurrences WHERE seq = ? AND status = 'queued' AND record IS NULL AND NOT EXISTS (SELECT 1 FROM events WHERE occurrence_id = ?)", seq, id).changes > 0;
|
|
6442
|
+
}
|
|
6443
|
+
return this.run(`DELETE FROM occurrences WHERE seq = ? AND status = 'queued' AND record IS NULL
|
|
6444
|
+
AND EXISTS (SELECT 1 FROM events WHERE seq = ? AND occurrence_id = ?)
|
|
6445
|
+
AND NOT EXISTS (SELECT 1 FROM events WHERE occurrence_id = ? AND seq > ?)`, seq, last, id, id, last).changes > 0;
|
|
6222
6446
|
}
|
|
6223
6447
|
async requeueRunning(lane) {
|
|
6224
6448
|
const rows = lane === undefined ? this.all("SELECT seq FROM occurrences WHERE status = 'running' ORDER BY seq") : this.all("SELECT seq FROM occurrences WHERE status = 'running' AND lane = ? ORDER BY seq", lane);
|
|
@@ -6258,6 +6482,10 @@ class SqliteStore {
|
|
|
6258
6482
|
const next = this.one("SELECT * FROM occurrences WHERE task_id = ? AND status = 'queued' AND due_at > ? ORDER BY due_at, created_at, seq LIMIT 1", taskId, now);
|
|
6259
6483
|
return { rows: rows.reverse().map(toOccurrence), total, next: next ? toOccurrence(next) : null };
|
|
6260
6484
|
}
|
|
6485
|
+
async runsNowSince(ownerId, since) {
|
|
6486
|
+
const row = this.one("SELECT COUNT(*) AS n FROM occurrences o JOIN tasks t ON o.task_id = 't' || t.seq WHERE t.owner_id = ? AND o.dedupe_key LIKE 'now:%' AND o.created_at >= ?", ownerId, since);
|
|
6487
|
+
return Number(row?.n ?? 0);
|
|
6488
|
+
}
|
|
6261
6489
|
async repliesTo(taskId, occurrenceIds, userId) {
|
|
6262
6490
|
if (occurrenceIds.length === 0)
|
|
6263
6491
|
return [];
|
|
@@ -6291,7 +6519,7 @@ class SqliteStore {
|
|
|
6291
6519
|
}
|
|
6292
6520
|
async listSeries(taskId, filter = {}) {
|
|
6293
6521
|
const points = filter.since === undefined ? this.all("SELECT * FROM series WHERE task_id = ? ORDER BY at, seq", taskId) : this.all("SELECT * FROM series WHERE task_id = ? AND at >= ? ORDER BY at, seq", taskId, filter.since);
|
|
6294
|
-
const kept = filter.limit === undefined ? points : points.slice(-filter.limit);
|
|
6522
|
+
const kept = filter.limit === undefined ? points : !(filter.limit >= 1) ? [] : points.slice(-filter.limit);
|
|
6295
6523
|
return kept.map(toPoint);
|
|
6296
6524
|
}
|
|
6297
6525
|
async addFinding(f) {
|
|
@@ -6366,10 +6594,10 @@ class SqliteStore {
|
|
|
6366
6594
|
WHERE occurrence_id = ? AND user_id = ? AND retry_at IS NOT NULL`, at, occurrenceId, userId);
|
|
6367
6595
|
return changes === 1 ? this.delivery(occurrenceId, userId) : null;
|
|
6368
6596
|
}
|
|
6369
|
-
async settleDelivery(occurrenceId, userId, settle) {
|
|
6597
|
+
async settleDelivery(occurrenceId, userId, settle, expected) {
|
|
6370
6598
|
const { changes } = this.run(`UPDATE deliveries SET status = ?, message_id = ?, error = ?, attempts = ?, deferrals = ?, retry_at = ?, settled_at = ?
|
|
6371
|
-
WHERE occurrence_id = ? AND user_id =
|
|
6372
|
-
if (changes === 0)
|
|
6599
|
+
WHERE occurrence_id = ? AND user_id = ?${expected === undefined ? "" : " AND status = ?"}`, settle.status, settle.messageId ?? null, settle.error ?? null, settle.attempts, settle.deferrals, settle.retryAt, settle.at, occurrenceId, userId, ...expected === undefined ? [] : [expected]);
|
|
6600
|
+
if (changes === 0 && !this.delivery(occurrenceId, userId))
|
|
6373
6601
|
throw new Error(`no delivery ${occurrenceId}:${userId}`);
|
|
6374
6602
|
return this.delivery(occurrenceId, userId);
|
|
6375
6603
|
}
|
|
@@ -6743,7 +6971,7 @@ function retimed(s, day, owner) {
|
|
|
6743
6971
|
return null;
|
|
6744
6972
|
}
|
|
6745
6973
|
async function heldRuns(store, task) {
|
|
6746
|
-
return (await store.listOccurrences({ taskId: task.id, status: "queued" })).filter((o) => !o.dedupeKey
|
|
6974
|
+
return (await store.listOccurrences({ taskId: task.id, status: "queued" })).filter((o) => !isOffSchedule(o.dedupeKey) && !hasFired(o));
|
|
6747
6975
|
}
|
|
6748
6976
|
async function restoreHeldRun(store, task, held, before, after, now) {
|
|
6749
6977
|
const s = task.schedule;
|
|
@@ -7233,7 +7461,7 @@ async function rescheduleOwned(d, owner, zoneChanged, before) {
|
|
|
7233
7461
|
return moved;
|
|
7234
7462
|
}
|
|
7235
7463
|
async function rescheduleKeepingSnoozes(d, task, owner, schedule, actorId) {
|
|
7236
|
-
const { task: updated, jobOut } = await reschedule(d.store, task, owner, schedule, actorId, d.clock.now());
|
|
7464
|
+
const { task: updated, jobOut } = await reschedule(d.store, task, owner, schedule, actorId, d.clock.now(), d.types[task.type]);
|
|
7237
7465
|
return { task: updated, jobOut };
|
|
7238
7466
|
}
|
|
7239
7467
|
var JOB_OUT_NOTE = "A run of this task is already with the model runner: it still counts, and the new schedule follows it.";
|
|
@@ -7397,6 +7625,10 @@ async function pauseLocked(d, user, taskId) {
|
|
|
7397
7625
|
const at = d.clock.now().toISOString();
|
|
7398
7626
|
await d.store.updateTask(task.id, { status: "paused", at });
|
|
7399
7627
|
await d.store.addTaskEvent({ taskId: task.id, actorId: user.id, kind: "paused", detail: "paused by the owner", at });
|
|
7628
|
+
for (const o of await d.store.listOccurrences({ taskId: task.id, status: "queued" })) {
|
|
7629
|
+
if (o.dedupeKey.startsWith(RUN_NOW_PREFIX))
|
|
7630
|
+
await deleteIdleRun(d.store, o);
|
|
7631
|
+
}
|
|
7400
7632
|
return { ok: true, text: `Paused \`${task.id}\`: nothing is sent until you resume it.` };
|
|
7401
7633
|
}
|
|
7402
7634
|
function resumeTask(d, user, taskId) {
|
|
@@ -7437,8 +7669,7 @@ async function deleteLocked(d, user, taskId) {
|
|
|
7437
7669
|
return { ok: false, error: NO_SUCH_TASK };
|
|
7438
7670
|
const at = d.clock.now().toISOString();
|
|
7439
7671
|
for (const o of await d.store.listOccurrences({ taskId: task.id, status: "queued" })) {
|
|
7440
|
-
|
|
7441
|
-
await d.store.deleteOccurrence(o.id);
|
|
7672
|
+
await deleteIdleRun(d.store, o);
|
|
7442
7673
|
}
|
|
7443
7674
|
await d.store.updateTask(task.id, { status: "archived", at });
|
|
7444
7675
|
await d.store.addTaskEvent({ taskId: task.id, actorId: user.id, kind: "archived", detail: "deleted by the owner", at });
|
|
@@ -7530,6 +7761,10 @@ function nearPattern(near) {
|
|
|
7530
7761
|
const literal = near.trim().replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
7531
7762
|
return `${literal}[^0-9]{0,40}([0-9](?:[0-9.,]{0,14}[0-9])?)`;
|
|
7532
7763
|
}
|
|
7764
|
+
function patternRefusal(pattern) {
|
|
7765
|
+
const problem = pattern === undefined ? null : patternProblem(pattern);
|
|
7766
|
+
return problem ? `That \`near\` text cannot be used: ${problem}` : null;
|
|
7767
|
+
}
|
|
7533
7768
|
function priceSettingsPlan(input) {
|
|
7534
7769
|
const hours = input.hours ?? DEFAULT_POLL_HOURS;
|
|
7535
7770
|
if (!Number.isInteger(hours) || hours < 1 || hours > MAX_POLL_HOURS)
|
|
@@ -7600,7 +7835,11 @@ async function startPrice(d, user, input) {
|
|
|
7600
7835
|
if (capped)
|
|
7601
7836
|
return { ok: false, error: capped };
|
|
7602
7837
|
const { name, hours, drop, baseline } = settings;
|
|
7603
|
-
const
|
|
7838
|
+
const pattern = near ? nearPattern(near) : undefined;
|
|
7839
|
+
const refused = patternRefusal(pattern);
|
|
7840
|
+
if (refused)
|
|
7841
|
+
return { ok: false, error: refused };
|
|
7842
|
+
const config = { url, dropPercent: drop, baseline, ...pattern !== undefined ? { pattern } : {} };
|
|
7604
7843
|
return { ok: true, pending: { url, title: name || titleFor(url), hours, config } };
|
|
7605
7844
|
}
|
|
7606
7845
|
async function trackPrice(d, user, input) {
|
|
@@ -9293,6 +9532,19 @@ var MIGRATIONS = [
|
|
|
9293
9532
|
ended_at TEXT
|
|
9294
9533
|
);
|
|
9295
9534
|
CREATE UNIQUE INDEX intake_conversations_open ON intake_conversations (user_id) WHERE status = 'open';
|
|
9535
|
+
`,
|
|
9536
|
+
`
|
|
9537
|
+
CREATE TABLE api_idempotency (
|
|
9538
|
+
user_id TEXT NOT NULL,
|
|
9539
|
+
key TEXT NOT NULL,
|
|
9540
|
+
request_hash TEXT NOT NULL,
|
|
9541
|
+
claim TEXT NOT NULL,
|
|
9542
|
+
status INTEGER,
|
|
9543
|
+
answer TEXT,
|
|
9544
|
+
created_at TEXT NOT NULL,
|
|
9545
|
+
PRIMARY KEY (user_id, key)
|
|
9546
|
+
);
|
|
9547
|
+
CREATE INDEX api_idempotency_created ON api_idempotency (created_at);
|
|
9296
9548
|
`
|
|
9297
9549
|
];
|
|
9298
9550
|
var BUSY_TIMEOUT_MS = 5000;
|
|
@@ -9930,6 +10182,7 @@ function scoutSchedule(every, zone, now) {
|
|
|
9930
10182
|
const start = `${w.year}-${String(w.month).padStart(2, "0")}-${String(w.day).padStart(2, "0")}`;
|
|
9931
10183
|
return { kind: "calendar", every, unit: "day", start };
|
|
9932
10184
|
}
|
|
10185
|
+
var SCOUT_CAP = `You already have ${MAX_LIVE_SCOUTS} scouts, the most one person may. Delete or change one instead.`;
|
|
9933
10186
|
async function liveScouts(d, user) {
|
|
9934
10187
|
const active = await d.store.listTasks({ ownerId: user.id, status: "active", type: "scout" });
|
|
9935
10188
|
const paused = await d.store.listTasks({ ownerId: user.id, status: "paused", type: "scout" });
|
|
@@ -9946,7 +10199,7 @@ async function createScout(d, user, input) {
|
|
|
9946
10199
|
if (capped)
|
|
9947
10200
|
return { ok: false, error: capped };
|
|
9948
10201
|
if (await liveScouts(d, user) >= MAX_LIVE_SCOUTS) {
|
|
9949
|
-
return { ok: false, error:
|
|
10202
|
+
return { ok: false, error: SCOUT_CAP };
|
|
9950
10203
|
}
|
|
9951
10204
|
const now = d.clock.now();
|
|
9952
10205
|
const actor = { userId: user.id, admin: user.admin };
|
|
@@ -10104,6 +10357,8 @@ async function pressLocked(d, user, ref) {
|
|
|
10104
10357
|
} catch (err) {
|
|
10105
10358
|
if (err instanceof ReplyRefusedError)
|
|
10106
10359
|
return { ok: false, error: err.message };
|
|
10360
|
+
if (err instanceof ConsentError)
|
|
10361
|
+
return { ok: false, error: "That is not yours to answer." };
|
|
10107
10362
|
throw err;
|
|
10108
10363
|
}
|
|
10109
10364
|
}
|
|
@@ -10560,6 +10815,7 @@ class Roster {
|
|
|
10560
10815
|
del("web_sessions", "user_id = ?", userId);
|
|
10561
10816
|
del("web_login_tokens", "user_id = ?", userId);
|
|
10562
10817
|
del("api_tokens", "user_id = ?", userId);
|
|
10818
|
+
del("api_idempotency", "user_id = ?", userId);
|
|
10563
10819
|
del("notices", "key LIKE 'budget:person:' || ? || ':%'", userId);
|
|
10564
10820
|
del("notices", "key LIKE 'digest:' || ? || ':%'", userId);
|
|
10565
10821
|
del("ceiling_changes", "user_id = ?", userId);
|
|
@@ -10586,6 +10842,24 @@ function replyText(r) {
|
|
|
10586
10842
|
return `chose ${String(r.payload?.choice ?? "?")}`;
|
|
10587
10843
|
return r.kind === "opt_out" ? "opted out" : r.kind;
|
|
10588
10844
|
}
|
|
10845
|
+
function owed(r) {
|
|
10846
|
+
return r.retryAt !== null || r.status === "claimed" || r.status === "pending";
|
|
10847
|
+
}
|
|
10848
|
+
function deliveryView(o, rows, people, viewer, seesEveryone) {
|
|
10849
|
+
const shown = seesEveryone ? rows : rows.filter((r) => r.userId === viewer.id);
|
|
10850
|
+
const sends = shown.map((r) => ({ to: people.get(r.userId)?.displayName ?? r.userId, status: r.status, error: r.error ? clip2(r.error, 120) : null }));
|
|
10851
|
+
if (shown.length === 0) {
|
|
10852
|
+
const finished = o.status !== "queued" && o.status !== "running";
|
|
10853
|
+
return { state: finished ? "nothing_to_send" : "not_run", reason: null, sends };
|
|
10854
|
+
}
|
|
10855
|
+
const lost = shown.find((r) => r.status === "failed" && !owed(r));
|
|
10856
|
+
if (lost)
|
|
10857
|
+
return { state: "failed", reason: lost.error ? clip2(lost.error, 120) : null, sends };
|
|
10858
|
+
const unsure = shown.find((r) => r.status === "unconfirmed");
|
|
10859
|
+
if (unsure)
|
|
10860
|
+
return { state: "unconfirmed", reason: unsure.error ? clip2(unsure.error, 120) : null, sends };
|
|
10861
|
+
return { state: shown.some(owed) ? "sending" : "delivered", reason: null, sends };
|
|
10862
|
+
}
|
|
10589
10863
|
var OTHER_RECIPIENT_ERROR = "a recipient cannot be messaged";
|
|
10590
10864
|
function runView(o, replies, people, viewer, now, seesEveryone) {
|
|
10591
10865
|
const error = o.status === "failed" && o.error ? seesEveryone || !errorNamesOther(o.error, viewer.id) ? clip2(o.error, 120) : OTHER_RECIPIENT_ERROR : null;
|
|
@@ -10660,8 +10934,11 @@ async function loadHistory(d, user, taskId) {
|
|
|
10660
10934
|
const tail = await d.store.changesTail(task.id, seesEveryone ? null : user.id, HISTORY_CHANGES);
|
|
10661
10935
|
const lead = tail.before?.kind === "paused" ? [tail.before] : [];
|
|
10662
10936
|
const changes = seesEveryone ? tail.rows : changesForRecipient([...lead, ...tail.rows], user.id).slice(lead.length);
|
|
10937
|
+
const deliveries = new Map;
|
|
10938
|
+
for (const o of runs.rows)
|
|
10939
|
+
deliveries.set(o.id, await d.store.listDeliveries({ occurrenceId: o.id }));
|
|
10663
10940
|
const people = new Map;
|
|
10664
|
-
for (const r of replies) {
|
|
10941
|
+
for (const r of [...replies, ...[...deliveries.values()].flat()]) {
|
|
10665
10942
|
if (!people.has(r.userId)) {
|
|
10666
10943
|
const u = await d.store.getUser(r.userId);
|
|
10667
10944
|
if (u)
|
|
@@ -10684,7 +10961,7 @@ async function loadHistory(d, user, taskId) {
|
|
|
10684
10961
|
task,
|
|
10685
10962
|
cadence: task.schedule && owner ? describeSchedule(task.schedule, owner, user.timeZone, now) : "no schedule",
|
|
10686
10963
|
next: next ? formatInstant(next.dueAt, user.timeZone, now) : null,
|
|
10687
|
-
runs: runs.rows.map((o) => runView(o, replies, people, user, now, seesEveryone)),
|
|
10964
|
+
runs: runs.rows.map((o) => ({ ...runView(o, replies, people, user, now, seesEveryone), delivery: deliveryView(o, deliveries.get(o.id) ?? [], people, user, seesEveryone) })),
|
|
10688
10965
|
earlierRuns: Math.max(0, runs.total - HISTORY_RUNS),
|
|
10689
10966
|
changes: changes.map((e) => changeView(e, user, now, names)),
|
|
10690
10967
|
earlierChanges: Math.max(0, tail.total - HISTORY_CHANGES),
|
|
@@ -11221,6 +11498,52 @@ async function loadUsage(store, now, days = USAGE_DAYS) {
|
|
|
11221
11498
|
return usageReport(rows, now, days);
|
|
11222
11499
|
}
|
|
11223
11500
|
|
|
11501
|
+
// plugins/tracker/src/run-now.ts
|
|
11502
|
+
var RUNNABLE = new Set(["reminder", "price", "wantlist", "wantjudge", "scout"]);
|
|
11503
|
+
var RUN_NOW_PER_DAY = 10;
|
|
11504
|
+
var DAY_MS4 = 24 * 3600000;
|
|
11505
|
+
var NOT_NOW = "That task cannot run now:";
|
|
11506
|
+
function runNow(d, user, taskId) {
|
|
11507
|
+
return d.locks.turn(taskId.trim(), () => runNowLocked(d, user, taskId), () => ({ ok: false, error: TASK_BUSY }));
|
|
11508
|
+
}
|
|
11509
|
+
async function runNowLocked(d, user, taskId) {
|
|
11510
|
+
const task = await ownLiveTask(d, user, taskId);
|
|
11511
|
+
if (!task)
|
|
11512
|
+
return { ok: false, error: NO_SUCH_TASK };
|
|
11513
|
+
if (task.status !== "active")
|
|
11514
|
+
return { ok: false, error: `That task is ${task.status}, not active.` };
|
|
11515
|
+
if (task.type === "renewal")
|
|
11516
|
+
return { ok: false, error: `${NOT_NOW} a renewal asks about its own date, at its own time.` };
|
|
11517
|
+
if (task.type === "research")
|
|
11518
|
+
return { ok: false, error: `${NOT_NOW} a research request runs once.` };
|
|
11519
|
+
if (!RUNNABLE.has(task.type))
|
|
11520
|
+
return { ok: false, error: `${NOT_NOW} a ${task.type} runs only on its own schedule.` };
|
|
11521
|
+
if (!task.schedule || task.schedule.kind === "once") {
|
|
11522
|
+
return { ok: false, error: `${NOT_NOW} a one-off runs once, at its own time; change its time instead.` };
|
|
11523
|
+
}
|
|
11524
|
+
const now = d.clock.now();
|
|
11525
|
+
const queued = await d.store.listOccurrences({ taskId: task.id, status: "queued" });
|
|
11526
|
+
if (queued.some((o) => !hasFired(o) && Date.parse(o.dueAt) <= now.getTime())) {
|
|
11527
|
+
return { ok: false, error: `${NOT_NOW} it already has a run due, which is on its way.` };
|
|
11528
|
+
}
|
|
11529
|
+
if (queued.some((o) => !hasFired(o) && o.dedupeKey.startsWith(FOLLOW_UP_PREFIX))) {
|
|
11530
|
+
return { ok: false, error: `${NOT_NOW} it is waiting to try its last look again.` };
|
|
11531
|
+
}
|
|
11532
|
+
const since = new Date(now.getTime() - DAY_MS4).toISOString();
|
|
11533
|
+
if (await d.store.runsNowSince(user.id, since) >= RUN_NOW_PER_DAY) {
|
|
11534
|
+
return { ok: false, error: `You already ran ${RUN_NOW_PER_DAY} tasks now in the last 24 hours; each runs on its own schedule meanwhile.` };
|
|
11535
|
+
}
|
|
11536
|
+
const at = now.toISOString();
|
|
11537
|
+
const made = await d.store.createOccurrence({ taskId: task.id, lane: task.lane, dueAt: at, dedupeKey: runNowKey(task.id, crypto.randomUUID()), at });
|
|
11538
|
+
if (!made)
|
|
11539
|
+
return { ok: false, error: TASK_BUSY };
|
|
11540
|
+
d.log.info(`run now: ${task.id} for ${user.id} (${made.id})`);
|
|
11541
|
+
return { ok: true, text: `Running \`${task.id}\` now: ${when2(task)}, and its schedule stays as it was.` };
|
|
11542
|
+
}
|
|
11543
|
+
function when2(task) {
|
|
11544
|
+
return task.lane === "execute" ? "it goes to the model runner on its next turn (a daily ceiling met holds it until midnight Eastern)" : "it goes within a minute";
|
|
11545
|
+
}
|
|
11546
|
+
|
|
11224
11547
|
// node_modules/@rackbops/styles/rackbops-noir/bundle.css
|
|
11225
11548
|
var bundle_default = `:where([data-rb-style], [data-rb-style] *), :where([data-rb-style], [data-rb-style] *):before, :where([data-rb-style], [data-rb-style] *):after {
|
|
11226
11549
|
box-sizing: border-box;
|
|
@@ -12565,7 +12888,7 @@ function loginPage(base, input) {
|
|
|
12565
12888
|
</section>`
|
|
12566
12889
|
});
|
|
12567
12890
|
}
|
|
12568
|
-
function
|
|
12891
|
+
function when3(entry, viewer, now) {
|
|
12569
12892
|
return entry.next ? formatInstant(entry.next.dueAt, viewer.timeZone, now) : "nothing due";
|
|
12570
12893
|
}
|
|
12571
12894
|
function cadence(entry, viewer, now) {
|
|
@@ -12588,7 +12911,7 @@ function tasksPage(v, data, now, flash = null, research = false) {
|
|
|
12588
12911
|
const rows = data.entries.map((e) => html`<tr>
|
|
12589
12912
|
<td><a class="rb-link" href="${v.base}/tasks/${e.task.id}">${e.task.title}</a></td>
|
|
12590
12913
|
<td>${e.from ? name3(e.from) : "you"}</td>
|
|
12591
|
-
<td>${
|
|
12914
|
+
<td>${when3(e, v.user, now)}</td>
|
|
12592
12915
|
<td>${cadence(e, v.user, now)}</td>
|
|
12593
12916
|
</tr>`);
|
|
12594
12917
|
const active = data.entries.length === 0 ? html`<p class="rb-muted">You have no active tasks.</p>` : html`<div class="rb-table-scroll"><table class="rb-table">
|
|
@@ -13117,6 +13440,31 @@ function editValues(d, user, task) {
|
|
|
13117
13440
|
return { name: task.title, hours: String(every), drop: String(c.dropPercent ?? 10), baseline: c.baseline ?? "last" };
|
|
13118
13441
|
}
|
|
13119
13442
|
|
|
13443
|
+
// plugins/tracker/src/web/task-version.ts
|
|
13444
|
+
function taskVersion(task) {
|
|
13445
|
+
const owned = { title: task.title, config: task.config, schedule: task.schedule, status: task.status, capabilities: task.capabilities };
|
|
13446
|
+
return sha256(canonicalJson(owned)).slice(0, 16);
|
|
13447
|
+
}
|
|
13448
|
+
function canonicalJson(value) {
|
|
13449
|
+
if (Array.isArray(value))
|
|
13450
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
13451
|
+
if (value !== null && typeof value === "object") {
|
|
13452
|
+
const o = value;
|
|
13453
|
+
return `{${Object.keys(o).sort().map((k) => `${JSON.stringify(k)}:${canonicalJson(o[k])}`).join(",")}}`;
|
|
13454
|
+
}
|
|
13455
|
+
return JSON.stringify(value);
|
|
13456
|
+
}
|
|
13457
|
+
function etag(version) {
|
|
13458
|
+
return `"${version}"`;
|
|
13459
|
+
}
|
|
13460
|
+
function ifMatchVersion(header) {
|
|
13461
|
+
if (header === null)
|
|
13462
|
+
return null;
|
|
13463
|
+
const m = /^\s*"([0-9a-f]{16})"\s*$/.exec(header);
|
|
13464
|
+
return m?.[1] ?? null;
|
|
13465
|
+
}
|
|
13466
|
+
var VERSION_MISMATCH = "That task has changed since you read it. Read it again (GET the task) and send its new version in If-Match.";
|
|
13467
|
+
|
|
13120
13468
|
// plugins/tracker/src/web/editor.ts
|
|
13121
13469
|
var READING = "I am still reading a page for your last new price tracker or want-list watch; try again when it is done.";
|
|
13122
13470
|
var GONE = NO_LONGER_LISTED;
|
|
@@ -13124,7 +13472,14 @@ var writer = (e) => ({ d: e.d, queue: e.queue, reading: e.reading, userId: e.v.u
|
|
|
13124
13472
|
function asUser(w, fn) {
|
|
13125
13473
|
return w.queue(async () => {
|
|
13126
13474
|
const user = await w.d.store.getUser(w.userId);
|
|
13127
|
-
|
|
13475
|
+
if (!user)
|
|
13476
|
+
return { ok: false, error: GONE };
|
|
13477
|
+
if (w.ifMatch) {
|
|
13478
|
+
const task = await w.d.store.getTask(w.ifMatch.taskId);
|
|
13479
|
+
if (!task || taskVersion(task) !== w.ifMatch.version)
|
|
13480
|
+
return { ok: false, error: VERSION_MISMATCH };
|
|
13481
|
+
}
|
|
13482
|
+
return fn(user);
|
|
13128
13483
|
});
|
|
13129
13484
|
}
|
|
13130
13485
|
var WEB_READ_MS = 8000;
|
|
@@ -13217,6 +13572,9 @@ function saveEdit(w, task, form) {
|
|
|
13217
13572
|
return asUser(w, (u) => editWant(w.d, u, task.id, wantEdit(form)));
|
|
13218
13573
|
return asUser(w, (u) => editPrice(w.d, u, task.id, priceEdit(form)));
|
|
13219
13574
|
}
|
|
13575
|
+
function runOn(w, taskId) {
|
|
13576
|
+
return asUser(w, (u) => runNow(w.d, u, taskId));
|
|
13577
|
+
}
|
|
13220
13578
|
function actOn(w, taskId, action) {
|
|
13221
13579
|
const act = action === "pause" ? pauseTask : action === "resume" ? resumeTask : deleteTask;
|
|
13222
13580
|
return asUser(w, (u) => act(w.d, u, taskId));
|
|
@@ -13381,12 +13739,16 @@ function refusal(error) {
|
|
|
13381
13739
|
return problem(503, "slow_page", error, { "Retry-After": "10" });
|
|
13382
13740
|
if (error === NOT_EDITABLE)
|
|
13383
13741
|
return problem(409, "conflict", error);
|
|
13742
|
+
if (error === VERSION_MISMATCH)
|
|
13743
|
+
return problem(412, "version_mismatch", error);
|
|
13384
13744
|
if (error === RESEARCH_OFF || error === SCOUT_OFF)
|
|
13385
13745
|
return problem(503, "unavailable", error);
|
|
13386
13746
|
if (error === FINISHED || /^That task is [a-z]+, not [a-z]+\.$/.test(error))
|
|
13387
13747
|
return problem(409, "conflict", error);
|
|
13388
|
-
if (/^You already (have|track|watch for) [0-9]+ /.test(error))
|
|
13748
|
+
if (/^You already (have|track|watch for|ran) [0-9]+ /.test(error))
|
|
13389
13749
|
return problem(409, "limit_reached", error);
|
|
13750
|
+
if (error.startsWith(NOT_NOW))
|
|
13751
|
+
return problem(409, "cannot_run_now", error);
|
|
13390
13752
|
if (error === INBOX_OFF || error === JUDGE_OFF)
|
|
13391
13753
|
return problem(503, "unavailable", error);
|
|
13392
13754
|
if (/ (is|are) not available on this bot\.$/.test(error))
|
|
@@ -13429,6 +13791,7 @@ async function taskJson(d, user, task) {
|
|
|
13429
13791
|
type: task.type,
|
|
13430
13792
|
title: task.title,
|
|
13431
13793
|
status: task.status === "archived" ? "deleted" : task.status,
|
|
13794
|
+
version: taskVersion(task),
|
|
13432
13795
|
cadence: task.schedule ? describeSchedule(task.schedule, user, user.timeZone, d.clock.now()) : "no schedule",
|
|
13433
13796
|
nextAt: await nextAt(d, task),
|
|
13434
13797
|
createdAt: task.createdAt,
|
|
@@ -13447,12 +13810,74 @@ function watchJson(c) {
|
|
|
13447
13810
|
...site ? { site, ...c.search !== undefined ? { search: c.search } : {}, ...c.game !== undefined ? { game: c.game } : {} } : {}
|
|
13448
13811
|
};
|
|
13449
13812
|
}
|
|
13450
|
-
|
|
13451
|
-
|
|
13813
|
+
var MAX_PAGE = 200;
|
|
13814
|
+
var cursorOf = (taskId) => Buffer.from(taskId).toString("base64url");
|
|
13815
|
+
var CURSOR = /^[A-Za-z0-9_-]{1,64}$/;
|
|
13816
|
+
async function listAnswer(d, user, query = new URLSearchParams) {
|
|
13817
|
+
const rawLimit = query.get("limit");
|
|
13818
|
+
const rawCursor = query.get("cursor");
|
|
13819
|
+
for (const key of ["limit", "cursor"]) {
|
|
13820
|
+
if (query.getAll(key).length > 1)
|
|
13821
|
+
return problem(400, "invalid", `\`${key}\` is given more than once.`);
|
|
13822
|
+
}
|
|
13823
|
+
let limit = null;
|
|
13824
|
+
if (rawLimit !== null) {
|
|
13825
|
+
limit = /^[0-9]{1,4}$/.test(rawLimit) ? Number(rawLimit) : 0;
|
|
13826
|
+
if (limit < 1 || limit > MAX_PAGE)
|
|
13827
|
+
return problem(400, "invalid", `\`limit\` is a whole number from 1 to ${MAX_PAGE}.`);
|
|
13828
|
+
}
|
|
13829
|
+
if (rawCursor !== null && limit === null)
|
|
13830
|
+
return problem(400, "invalid", "`cursor` goes with `limit`.");
|
|
13831
|
+
const all = await d.store.listTasks({ ownerId: user.id });
|
|
13832
|
+
let start = 0;
|
|
13833
|
+
if (rawCursor !== null) {
|
|
13834
|
+
const at = CURSOR.test(rawCursor) ? all.findIndex((t) => cursorOf(t.id) === rawCursor) : -1;
|
|
13835
|
+
if (at < 0)
|
|
13836
|
+
return problem(400, "invalid_cursor", "That cursor is not one this API gave you: start again without it.");
|
|
13837
|
+
start = at + 1;
|
|
13838
|
+
}
|
|
13452
13839
|
const out = [];
|
|
13453
|
-
|
|
13840
|
+
let last = null;
|
|
13841
|
+
let more = false;
|
|
13842
|
+
for (let i = start;i < all.length; i++) {
|
|
13843
|
+
const t = all[i];
|
|
13844
|
+
if (t.status === "archived")
|
|
13845
|
+
continue;
|
|
13846
|
+
if (limit !== null && out.length === limit) {
|
|
13847
|
+
more = true;
|
|
13848
|
+
break;
|
|
13849
|
+
}
|
|
13454
13850
|
out.push(await taskJson(d, user, t));
|
|
13455
|
-
|
|
13851
|
+
last = t;
|
|
13852
|
+
}
|
|
13853
|
+
return { status: 200, body: { tasks: out, ...limit !== null ? { nextCursor: more && last ? cursorOf(last.id) : null } : {} } };
|
|
13854
|
+
}
|
|
13855
|
+
function capabilitiesAnswer(d, user, rate) {
|
|
13856
|
+
return {
|
|
13857
|
+
status: 200,
|
|
13858
|
+
body: {
|
|
13859
|
+
types: intakeTypes(d),
|
|
13860
|
+
owner: { timeZone: user.timeZone, preferredHour: user.preferredHour },
|
|
13861
|
+
limits: {
|
|
13862
|
+
liveTasks: MAX_LIVE_TASKS,
|
|
13863
|
+
priceTrackers: MAX_PRICE_TASKS,
|
|
13864
|
+
researchWaiting: MAX_LIVE_RESEARCH,
|
|
13865
|
+
scouts: MAX_LIVE_SCOUTS,
|
|
13866
|
+
wantWatches: MAX_WANT_TASKS,
|
|
13867
|
+
pageSize: MAX_PAGE,
|
|
13868
|
+
requestsBurst: rate.burst,
|
|
13869
|
+
requestsPerSecond: rate.perSecond
|
|
13870
|
+
},
|
|
13871
|
+
sources: {
|
|
13872
|
+
runner: d.research === true,
|
|
13873
|
+
bgg: d.bgg === true,
|
|
13874
|
+
ebay: d.ebay === true,
|
|
13875
|
+
shopify: d.shopify === true,
|
|
13876
|
+
mail: d.mail != null,
|
|
13877
|
+
webArea: d.webEditor
|
|
13878
|
+
}
|
|
13879
|
+
}
|
|
13880
|
+
};
|
|
13456
13881
|
}
|
|
13457
13882
|
async function getAnswer(d, user, id) {
|
|
13458
13883
|
const task = await ownTask(d, user, id);
|
|
@@ -13463,6 +13888,7 @@ async function getAnswer(d, user, id) {
|
|
|
13463
13888
|
return notFound();
|
|
13464
13889
|
return {
|
|
13465
13890
|
status: 200,
|
|
13891
|
+
headers: { ETag: etag(taskVersion(view.task)) },
|
|
13466
13892
|
body: {
|
|
13467
13893
|
task: await taskJson(d, user, view.task),
|
|
13468
13894
|
history: { next: view.next, runs: view.runs, earlierRuns: view.earlierRuns, changes: view.changes, earlierChanges: view.earlierChanges },
|
|
@@ -13474,10 +13900,11 @@ async function written(w, user, result, status, base) {
|
|
|
13474
13900
|
if (!result.ok)
|
|
13475
13901
|
return refusal(result.error);
|
|
13476
13902
|
const task = await w.d.store.getTask(result.task.id) ?? result.task;
|
|
13903
|
+
const tag = { ETag: etag(taskVersion(task)) };
|
|
13477
13904
|
return {
|
|
13478
13905
|
status,
|
|
13479
13906
|
body: { task: await taskJson(w.d, user, task), message: result.text },
|
|
13480
|
-
|
|
13907
|
+
headers: status === 201 ? { ...tag, Location: `${base}/tasks/${encodeURIComponent(task.id)}` } : tag
|
|
13481
13908
|
};
|
|
13482
13909
|
}
|
|
13483
13910
|
async function createAnswer(w, user, body, base) {
|
|
@@ -13494,6 +13921,9 @@ async function createAnswer(w, user, body, base) {
|
|
|
13494
13921
|
return form;
|
|
13495
13922
|
return written(w, user, await makeTask(w, type, form), 201, base);
|
|
13496
13923
|
}
|
|
13924
|
+
function bound(w, task) {
|
|
13925
|
+
return w.ifMatch ? { ...w, ifMatch: { taskId: task.id, version: w.ifMatch.version } } : w;
|
|
13926
|
+
}
|
|
13497
13927
|
async function editAnswer(w, user, id, body, base) {
|
|
13498
13928
|
const task = await editableTask(w.d, user, id);
|
|
13499
13929
|
if (!task)
|
|
@@ -13501,17 +13931,27 @@ async function editAnswer(w, user, id, body, base) {
|
|
|
13501
13931
|
const form = asFields(body, task.type, "edit");
|
|
13502
13932
|
if (!(form instanceof URLSearchParams))
|
|
13503
13933
|
return form;
|
|
13504
|
-
return written(w, user, await saveEdit(w, task, form), 200, base);
|
|
13934
|
+
return written(w, user, await saveEdit(bound(w, task), task, form), 200, base);
|
|
13935
|
+
}
|
|
13936
|
+
async function runNowAnswer(w, user, id) {
|
|
13937
|
+
const task = await ownTask(w.d, user, id);
|
|
13938
|
+
if (!task || task.status === "archived")
|
|
13939
|
+
return notFound();
|
|
13940
|
+
const done = await runOn(w, task.id);
|
|
13941
|
+
if (!done.ok)
|
|
13942
|
+
return refusal(done.error);
|
|
13943
|
+
const after = await w.d.store.getTask(task.id) ?? task;
|
|
13944
|
+
return { status: 202, headers: { ETag: etag(taskVersion(after)) }, body: { task: await taskJson(w.d, user, after), message: done.text } };
|
|
13505
13945
|
}
|
|
13506
13946
|
async function actAnswer(w, user, id, action) {
|
|
13507
13947
|
const task = await ownTask(w.d, user, id);
|
|
13508
13948
|
if (!task || task.status === "archived")
|
|
13509
13949
|
return notFound();
|
|
13510
|
-
const done = await actOn(w, task.id, action);
|
|
13950
|
+
const done = await actOn(bound(w, task), task.id, action);
|
|
13511
13951
|
if (!done.ok)
|
|
13512
13952
|
return refusal(done.error);
|
|
13513
13953
|
const after = await w.d.store.getTask(task.id) ?? task;
|
|
13514
|
-
return { status: 200, body: { task: await taskJson(w.d, user, after), message: done.text } };
|
|
13954
|
+
return { status: 200, headers: { ETag: etag(taskVersion(after)) }, body: { task: await taskJson(w.d, user, after), message: done.text } };
|
|
13515
13955
|
}
|
|
13516
13956
|
|
|
13517
13957
|
// plugins/tracker/src/intake.ts
|
|
@@ -13996,7 +14436,7 @@ var MAX_TOKENS_PER_PERSON = 10;
|
|
|
13996
14436
|
var MAX_TOKEN_NAME = 50;
|
|
13997
14437
|
var TOKEN_EXPIRY_DAYS = [30, 90, 365];
|
|
13998
14438
|
var DEFAULT_EXPIRY_DAYS = 90;
|
|
13999
|
-
var
|
|
14439
|
+
var DAY_MS5 = 24 * 60 * 60 * 1000;
|
|
14000
14440
|
var TOUCH_EVERY_MS = 60 * 1000;
|
|
14001
14441
|
function isApiTokenShape(value) {
|
|
14002
14442
|
return /^trk_[A-Za-z0-9_-]{43}$/.test(value);
|
|
@@ -14048,7 +14488,7 @@ class ApiTokens {
|
|
|
14048
14488
|
return { ok: false, error: `You already have ${MAX_TOKENS_PER_PERSON} API tokens, the most one person may. Revoke one first.` };
|
|
14049
14489
|
}
|
|
14050
14490
|
const token = `${TOKEN_PREFIX}${randomToken()}`;
|
|
14051
|
-
const expiresAt = new Date(now.getTime() + input.days *
|
|
14491
|
+
const expiresAt = new Date(now.getTime() + input.days * DAY_MS5).toISOString();
|
|
14052
14492
|
const r = this.db.query(`INSERT INTO api_tokens (token_hash, user_id, name, created_at, expires_at, last_used_at, member_checked_at)
|
|
14053
14493
|
VALUES (?, ?, ?, ?, ?, NULL, ?) RETURNING ${COLUMNS}`).get(sha256(token), userId, name, at, expiresAt, memberCheckedAt);
|
|
14054
14494
|
return { ok: true, token, row: toToken(r) };
|
|
@@ -14624,7 +15064,7 @@ function startUsrRun(a, usr, me, kind) {
|
|
|
14624
15064
|
const current = usrLinkRuns.get(usr);
|
|
14625
15065
|
if (current?.running)
|
|
14626
15066
|
return { ok: false, text: `Already ${RUN_WORDS[current.kind].doing}: ${current.done} of ${current.total} done.` };
|
|
14627
|
-
const waiting = a.d.roster.people().filter((p) => p.discordId !== null &&
|
|
15067
|
+
const waiting = a.d.roster.people().filter((p) => p.discordId !== null && wanted2(kind, usr.links.subjectOf(p.id)));
|
|
14628
15068
|
if (waiting.length === 0)
|
|
14629
15069
|
return { ok: true, text: RUN_WORDS[kind].none };
|
|
14630
15070
|
const run = { kind, running: true, done: 0, total: waiting.length, result: null };
|
|
@@ -14642,7 +15082,7 @@ function startUsrRun(a, usr, me, kind) {
|
|
|
14642
15082
|
function personsOwn400(err) {
|
|
14643
15083
|
return err.status === 400 && /^(discord_user_id|display_name)\b/.test(err.reason);
|
|
14644
15084
|
}
|
|
14645
|
-
function
|
|
15085
|
+
function wanted2(kind, subject) {
|
|
14646
15086
|
return kind === "link" ? subject === null : subject !== null;
|
|
14647
15087
|
}
|
|
14648
15088
|
async function usrRun(a, usr, invoker, by, guildId, ids, run) {
|
|
@@ -14667,7 +15107,7 @@ async function usrRun(a, usr, invoker, by, guildId, ids, run) {
|
|
|
14667
15107
|
const notAdmin = () => end(false, `${words.stopped}: ${by}, who started it, is no longer an admin here. ${soFar()}.`);
|
|
14668
15108
|
const stillWaiting = (id) => fresh(a, true, async () => {
|
|
14669
15109
|
const p = await a.d.store.getUser(id);
|
|
14670
|
-
return p && p.discordId !== null &&
|
|
15110
|
+
return p && p.discordId !== null && wanted2(run.kind, usr.links.subjectOf(id)) ? p : "skip";
|
|
14671
15111
|
});
|
|
14672
15112
|
for (const id of ids) {
|
|
14673
15113
|
const first = await stillWaiting(id);
|
|
@@ -14755,6 +15195,46 @@ async function forgetRoute(a, method, form) {
|
|
|
14755
15195
|
return redirect(`${base}/signin?forgotten=1`, [a.signOutCookie]);
|
|
14756
15196
|
}
|
|
14757
15197
|
|
|
15198
|
+
// plugins/tracker/src/web/api-delegation.ts
|
|
15199
|
+
var DELEGATES_ENV = "TRACKER_API_DELEGATES";
|
|
15200
|
+
var DELEGATES_FORMAT = "^[0-9]+(,[0-9]+)*$";
|
|
15201
|
+
function parseDelegates(env, usrOn) {
|
|
15202
|
+
const raw = env[DELEGATES_ENV]?.trim();
|
|
15203
|
+
if (raw === undefined || raw === "")
|
|
15204
|
+
return new Set;
|
|
15205
|
+
if (!usrOn)
|
|
15206
|
+
throw new Error(`${DELEGATES_ENV} needs TRACKER_USR_URL: delegation tokens come from usr`);
|
|
15207
|
+
if (!new RegExp(DELEGATES_FORMAT).test(raw))
|
|
15208
|
+
throw new Error(`${DELEGATES_ENV} is usr API key ids separated by commas, such as 12,31`);
|
|
15209
|
+
return new Set(raw.split(",").map((id) => `key:${Number(id)}`));
|
|
15210
|
+
}
|
|
15211
|
+
function operationOf(r, method) {
|
|
15212
|
+
switch (r.kind) {
|
|
15213
|
+
case "me":
|
|
15214
|
+
return method === "GET" ? "whoami" : null;
|
|
15215
|
+
case "types":
|
|
15216
|
+
return method === "GET" ? "list_task_types" : null;
|
|
15217
|
+
case "capabilities":
|
|
15218
|
+
return method === "GET" ? "get_capabilities" : null;
|
|
15219
|
+
case "validate":
|
|
15220
|
+
return method === "POST" ? "validate_task" : null;
|
|
15221
|
+
case "tasks":
|
|
15222
|
+
return method === "GET" ? "list_tasks" : method === "POST" ? "create_task" : null;
|
|
15223
|
+
case "task":
|
|
15224
|
+
return method === "GET" ? "get_task" : method === "PATCH" ? "update_task" : method === "DELETE" ? "delete_task" : null;
|
|
15225
|
+
case "act":
|
|
15226
|
+
return method === "POST" ? r.action === "pause" ? "pause_task" : "resume_task" : null;
|
|
15227
|
+
case "run":
|
|
15228
|
+
return method === "POST" ? "run_task" : null;
|
|
15229
|
+
case "listings":
|
|
15230
|
+
return method === "POST" ? "send_listings" : null;
|
|
15231
|
+
case "sources":
|
|
15232
|
+
return method === "POST" ? "send_sources" : null;
|
|
15233
|
+
case "ready":
|
|
15234
|
+
return method === "POST" ? "sources_ready" : null;
|
|
15235
|
+
}
|
|
15236
|
+
}
|
|
15237
|
+
|
|
14758
15238
|
// plugins/tracker/src/web/api-listings.ts
|
|
14759
15239
|
var EBAY_DOMAINS = [
|
|
14760
15240
|
"ebay.com",
|
|
@@ -14930,7 +15410,7 @@ async function sourcesReadyAnswer(w, user, id) {
|
|
|
14930
15410
|
if (followUp)
|
|
14931
15411
|
return fullerPass(w, user, task, owner, followUp, held, now);
|
|
14932
15412
|
try {
|
|
14933
|
-
await reschedule(w.d.store, task, owner, { kind: "once", at: now.toISOString() }, user.id, now);
|
|
15413
|
+
await reschedule(w.d.store, task, owner, { kind: "once", at: now.toISOString() }, user.id, now, w.d.types[task.type]);
|
|
14934
15414
|
} catch (err) {
|
|
14935
15415
|
if (err instanceof ScheduleError)
|
|
14936
15416
|
return problem(409, "conflict", "That research request has started.");
|
|
@@ -14946,7 +15426,7 @@ async function fullerPass(w, user, task, owner, followUp, held, now) {
|
|
|
14946
15426
|
const store = w.d.store;
|
|
14947
15427
|
let next;
|
|
14948
15428
|
try {
|
|
14949
|
-
next = (await reschedule(store, task, owner, { kind: "once", at: now.toISOString() }, user.id, now)).next;
|
|
15429
|
+
next = (await reschedule(store, task, owner, { kind: "once", at: now.toISOString() }, user.id, now, w.d.types[task.type])).next;
|
|
14950
15430
|
} catch (err) {
|
|
14951
15431
|
if (err instanceof ScheduleError)
|
|
14952
15432
|
return problem(409, "conflict", "That research request has started.");
|
|
@@ -14954,7 +15434,7 @@ async function fullerPass(w, user, task, owner, followUp, held, now) {
|
|
|
14954
15434
|
}
|
|
14955
15435
|
if (!next)
|
|
14956
15436
|
return problem(409, "conflict", "That research request could not be started just now; try again later.");
|
|
14957
|
-
if (!await store
|
|
15437
|
+
if (!await deleteIdleRun(store, followUp)) {
|
|
14958
15438
|
await store.deleteOccurrence(next.id);
|
|
14959
15439
|
return problem(409, "conflict", "That research request has started.");
|
|
14960
15440
|
}
|
|
@@ -14964,6 +15444,148 @@ async function fullerPass(w, user, task, owner, followUp, held, now) {
|
|
|
14964
15444
|
};
|
|
14965
15445
|
}
|
|
14966
15446
|
|
|
15447
|
+
// plugins/tracker/src/web/api-idempotency.ts
|
|
15448
|
+
import { randomBytes as randomBytes3 } from "crypto";
|
|
15449
|
+
var KEEP_MS = 24 * 60 * 60 * 1000;
|
|
15450
|
+
var PENDING_MS = 2 * 60 * 1000;
|
|
15451
|
+
function isIdempotencyKey(value) {
|
|
15452
|
+
return /^[A-Za-z0-9_.:-]{8,128}$/.test(value);
|
|
15453
|
+
}
|
|
15454
|
+
var OUTCOME_UNKNOWN = "That request failed partway and may have made its change. Read your tasks before you send it again, under a new Idempotency-Key.";
|
|
15455
|
+
|
|
15456
|
+
class ApiIdempotency {
|
|
15457
|
+
db;
|
|
15458
|
+
constructor(db) {
|
|
15459
|
+
this.db = db;
|
|
15460
|
+
}
|
|
15461
|
+
begin(userId, key, requestHash, now) {
|
|
15462
|
+
const at = now.toISOString();
|
|
15463
|
+
return this.db.transaction(() => {
|
|
15464
|
+
this.db.query("DELETE FROM api_idempotency WHERE created_at <= ?").run(new Date(now.getTime() - KEEP_MS).toISOString());
|
|
15465
|
+
const row = this.db.query("SELECT request_hash, status, answer, created_at FROM api_idempotency WHERE user_id = ? AND key = ?").get(userId, key);
|
|
15466
|
+
if (row !== null) {
|
|
15467
|
+
const stale = row.status === null && Date.parse(row.created_at) <= now.getTime() - PENDING_MS;
|
|
15468
|
+
if (!stale) {
|
|
15469
|
+
if (row.request_hash !== requestHash)
|
|
15470
|
+
return { kind: "conflict" };
|
|
15471
|
+
if (row.status === null || row.answer === null)
|
|
15472
|
+
return { kind: "in_progress" };
|
|
15473
|
+
const kept = JSON.parse(row.answer);
|
|
15474
|
+
return { kind: "replay", answer: { status: row.status, body: kept.body, headers: kept.headers } };
|
|
15475
|
+
}
|
|
15476
|
+
this.db.query("DELETE FROM api_idempotency WHERE user_id = ? AND key = ?").run(userId, key);
|
|
15477
|
+
}
|
|
15478
|
+
const claim = randomBytes3(16).toString("hex");
|
|
15479
|
+
this.db.query("INSERT INTO api_idempotency (user_id, key, request_hash, claim, status, answer, created_at) VALUES (?, ?, ?, ?, NULL, NULL, ?)").run(userId, key, requestHash, claim, at);
|
|
15480
|
+
return { kind: "new", claim };
|
|
15481
|
+
})();
|
|
15482
|
+
}
|
|
15483
|
+
finish(userId, key, claim, answer) {
|
|
15484
|
+
if (answer.status >= 200 && answer.status < 300)
|
|
15485
|
+
this.keep(userId, key, claim, answer);
|
|
15486
|
+
else
|
|
15487
|
+
this.abandon(userId, key, claim);
|
|
15488
|
+
}
|
|
15489
|
+
fail(userId, key, claim) {
|
|
15490
|
+
this.keep(userId, key, claim, { status: 500, body: { error: { code: "outcome_unknown", message: OUTCOME_UNKNOWN } }, headers: {} });
|
|
15491
|
+
}
|
|
15492
|
+
abandon(userId, key, claim) {
|
|
15493
|
+
this.db.query("DELETE FROM api_idempotency WHERE user_id = ? AND key = ? AND claim = ? AND status IS NULL").run(userId, key, claim);
|
|
15494
|
+
}
|
|
15495
|
+
keep(userId, key, claim, answer) {
|
|
15496
|
+
this.db.query("UPDATE api_idempotency SET status = ?, answer = ? WHERE user_id = ? AND key = ? AND claim = ? AND status IS NULL").run(answer.status, JSON.stringify({ body: answer.body, headers: answer.headers }), userId, key, claim);
|
|
15497
|
+
}
|
|
15498
|
+
}
|
|
15499
|
+
|
|
15500
|
+
// plugins/tracker/src/web/api-validate.ts
|
|
15501
|
+
function firstDueAfter2(schedule, now) {
|
|
15502
|
+
if (schedule.kind === "once")
|
|
15503
|
+
return new Date(0);
|
|
15504
|
+
if (schedule.kind === "poll")
|
|
15505
|
+
return new Date(Date.parse(schedule.start) - 1);
|
|
15506
|
+
return now;
|
|
15507
|
+
}
|
|
15508
|
+
function runs(schedule, user, now, renewal) {
|
|
15509
|
+
const ctx = { zone: user.timeZone, preferredHour: user.preferredHour };
|
|
15510
|
+
if (renewal) {
|
|
15511
|
+
const ask = nextDue(schedule, new Date(0), ctx);
|
|
15512
|
+
if (ask && ask.getTime() <= now.getTime())
|
|
15513
|
+
return { first: ask, second: nextDue(schedule, now, ctx) };
|
|
15514
|
+
}
|
|
15515
|
+
const first = nextDue(schedule, firstDueAfter2(schedule, now), ctx);
|
|
15516
|
+
return { first, second: first && schedule.kind !== "once" ? nextDue(schedule, first, ctx) : null };
|
|
15517
|
+
}
|
|
15518
|
+
async function validateAnswer(d, user, body) {
|
|
15519
|
+
const type = body.type;
|
|
15520
|
+
if (typeof type !== "string" || !NEW_TYPES.includes(type)) {
|
|
15521
|
+
return problem(400, "invalid", `\`type\` is one of ${offeredTypes(d).join(", ")}.`);
|
|
15522
|
+
}
|
|
15523
|
+
if (type === "research" && !d.research)
|
|
15524
|
+
return refusal(RESEARCH_OFF);
|
|
15525
|
+
if (type === "scout" && (!d.types.scout || !d.research))
|
|
15526
|
+
return refusal(SCOUT_OFF);
|
|
15527
|
+
if (type === "reminder" && !d.types.reminder)
|
|
15528
|
+
return refusal("Reminders are not available on this bot.");
|
|
15529
|
+
if (type === "renewal" && !d.types.renewal)
|
|
15530
|
+
return refusal("Renewals are not available on this bot.");
|
|
15531
|
+
const form = asFields(body, type, "new");
|
|
15532
|
+
if (!(form instanceof URLSearchParams))
|
|
15533
|
+
return form;
|
|
15534
|
+
const now = d.clock.now();
|
|
15535
|
+
let schedule = null;
|
|
15536
|
+
if (type === "reminder") {
|
|
15537
|
+
const plan = reminderPlan(d, user, reminderInput(form));
|
|
15538
|
+
if (!plan.ok)
|
|
15539
|
+
return refusal(plan.error);
|
|
15540
|
+
schedule = plan.schedule;
|
|
15541
|
+
} else if (type === "renewal") {
|
|
15542
|
+
const plan = renewalPlan(d, user, renewalInput(form));
|
|
15543
|
+
if (!plan.ok)
|
|
15544
|
+
return refusal(plan.error);
|
|
15545
|
+
schedule = plan.schedule;
|
|
15546
|
+
} else if (type === "scout") {
|
|
15547
|
+
const plan = scoutPlan(scoutInput(form));
|
|
15548
|
+
if (!plan.ok)
|
|
15549
|
+
return refusal(plan.error);
|
|
15550
|
+
schedule = scoutSchedule(plan.every, user.timeZone, now);
|
|
15551
|
+
}
|
|
15552
|
+
const capped = await liveTaskCap(d, user);
|
|
15553
|
+
if (capped)
|
|
15554
|
+
return refusal(capped);
|
|
15555
|
+
if (type === "scout" && await liveScouts(d, user) >= MAX_LIVE_SCOUTS) {
|
|
15556
|
+
return refusal(SCOUT_CAP);
|
|
15557
|
+
}
|
|
15558
|
+
if (schedule === null) {
|
|
15559
|
+
return {
|
|
15560
|
+
status: 200,
|
|
15561
|
+
body: {
|
|
15562
|
+
valid: true,
|
|
15563
|
+
type,
|
|
15564
|
+
checked: "fields",
|
|
15565
|
+
message: `The fields of that ${type} are right, and you are under the live-task cap. The rest is checked when it is made: ${type === "research" ? "the model runner takes it then" : "its page or source is read then"}. Nothing was made.`
|
|
15566
|
+
}
|
|
15567
|
+
};
|
|
15568
|
+
}
|
|
15569
|
+
const problems = scheduleProblems(schedule);
|
|
15570
|
+
if (problems.length > 0)
|
|
15571
|
+
return refusal(`That schedule does not work: ${problems.join("; ")}.`);
|
|
15572
|
+
const { first, second: then } = runs(schedule, user, now, type === "renewal");
|
|
15573
|
+
const cadence = describeSchedule(schedule, user, user.timeZone, now);
|
|
15574
|
+
return {
|
|
15575
|
+
status: 200,
|
|
15576
|
+
body: {
|
|
15577
|
+
valid: true,
|
|
15578
|
+
type,
|
|
15579
|
+
checked: "full",
|
|
15580
|
+
timeZone: user.timeZone,
|
|
15581
|
+
cadence,
|
|
15582
|
+
firstAt: first?.toISOString() ?? null,
|
|
15583
|
+
thenAt: then?.toISOString() ?? null,
|
|
15584
|
+
message: first ? `That ${type} would first run ${first.getTime() <= now.getTime() ? "at once" : formatInstant(first.toISOString(), user.timeZone, now)}${then ? `, then ${formatInstant(then.toISOString(), user.timeZone, now)}` : ""} (${cadence}). Nothing was made.` : `That ${type} would never run: its time has no next run. Nothing was made.`
|
|
15585
|
+
}
|
|
15586
|
+
};
|
|
15587
|
+
}
|
|
15588
|
+
|
|
14967
15589
|
// plugins/tracker/src/web/api.ts
|
|
14968
15590
|
var API_PREFIX = "/api/v1";
|
|
14969
15591
|
var MAX_API_BYTES = 16 * 1024;
|
|
@@ -15019,6 +15641,7 @@ function bearer(header) {
|
|
|
15019
15641
|
return m?.[1] ?? null;
|
|
15020
15642
|
}
|
|
15021
15643
|
var ACT = /^\/tasks\/([^/]+)\/(pause|resume)$/;
|
|
15644
|
+
var RUN = /^\/tasks\/([^/]+)\/run$/;
|
|
15022
15645
|
var LISTINGS = /^\/tasks\/([^/]+)\/listings$/;
|
|
15023
15646
|
var SOURCES = /^\/tasks\/([^/]+)\/sources(\/ready)?$/;
|
|
15024
15647
|
var ONE = /^\/tasks\/([^/]+)$/;
|
|
@@ -15034,11 +15657,18 @@ function route(rest) {
|
|
|
15034
15657
|
return { kind: "me" };
|
|
15035
15658
|
if (rest === "/types")
|
|
15036
15659
|
return { kind: "types" };
|
|
15660
|
+
if (rest === "/capabilities")
|
|
15661
|
+
return { kind: "capabilities" };
|
|
15037
15662
|
if (rest === "/tasks")
|
|
15038
15663
|
return { kind: "tasks" };
|
|
15664
|
+
if (rest === "/tasks/validate")
|
|
15665
|
+
return { kind: "validate" };
|
|
15039
15666
|
const act = ACT.exec(rest);
|
|
15040
15667
|
if (act)
|
|
15041
15668
|
return { kind: "act", id: segment(act[1]), action: act[2] };
|
|
15669
|
+
const run = RUN.exec(rest);
|
|
15670
|
+
if (run)
|
|
15671
|
+
return { kind: "run", id: segment(run[1]) };
|
|
15042
15672
|
const listings = LISTINGS.exec(rest);
|
|
15043
15673
|
if (listings)
|
|
15044
15674
|
return { kind: "listings", id: segment(listings[1]) };
|
|
@@ -15053,7 +15683,7 @@ function allowedMethods(r) {
|
|
|
15053
15683
|
return "GET, POST";
|
|
15054
15684
|
if (r.kind === "task")
|
|
15055
15685
|
return "GET, PATCH, DELETE";
|
|
15056
|
-
if (r.kind === "act" || r.kind === "listings" || r.kind === "sources" || r.kind === "ready")
|
|
15686
|
+
if (r.kind === "validate" || r.kind === "act" || r.kind === "run" || r.kind === "listings" || r.kind === "sources" || r.kind === "ready")
|
|
15057
15687
|
return "POST";
|
|
15058
15688
|
return "GET";
|
|
15059
15689
|
}
|
|
@@ -15084,6 +15714,10 @@ async function authenticate(w, request) {
|
|
|
15084
15714
|
const invalid = problem(401, "invalid_token", "That token is unknown, revoked or expired.", { "WWW-Authenticate": `${CHALLENGE}, error="invalid_token"` });
|
|
15085
15715
|
const sent = bearer(header);
|
|
15086
15716
|
const now = d.clock.now();
|
|
15717
|
+
const delegates = d.usr?.delegates;
|
|
15718
|
+
if (sent !== null && !sent.startsWith(TOKEN_PREFIX) && d.usr && delegates && delegates.size > 0) {
|
|
15719
|
+
return authenticateDelegation(w, d.usr, delegates, sent, invalid);
|
|
15720
|
+
}
|
|
15087
15721
|
const token = sent === null ? null : d.apiTokens.find(sent, now);
|
|
15088
15722
|
if (!token) {
|
|
15089
15723
|
const waitFailed = w.failed.take(FAILED_KEY, now.getTime());
|
|
@@ -15108,7 +15742,61 @@ async function authenticate(w, request) {
|
|
|
15108
15742
|
if (!current)
|
|
15109
15743
|
return no(invalid);
|
|
15110
15744
|
d.apiTokens.touch(current, d.clock.now());
|
|
15111
|
-
return { ok: true, token: current, user };
|
|
15745
|
+
return { ok: true, via: { kind: "token", token: current }, user };
|
|
15746
|
+
}
|
|
15747
|
+
async function authenticateDelegation(w, usr, delegates, sent, invalid) {
|
|
15748
|
+
const { d } = w;
|
|
15749
|
+
const no = (answer) => ({ ok: false, answer });
|
|
15750
|
+
const failed = () => {
|
|
15751
|
+
const wait = w.failed.take(FAILED_KEY, d.clock.now().getTime());
|
|
15752
|
+
return no(wait > 0 ? problem(429, "rate_limited", `Too many requests with bad tokens; try again in ${wait} s.`, { "Retry-After": String(wait) }) : invalid);
|
|
15753
|
+
};
|
|
15754
|
+
const checked = await usr.verifier.verifyDelegation(sent, usr.app);
|
|
15755
|
+
if (checked === "unavailable")
|
|
15756
|
+
return no(problem(503, "usr_unavailable", "I could not fetch usr's signing keys to check that token; try again in a minute.", { "Retry-After": "60" }));
|
|
15757
|
+
if (checked === "invalid" || !delegates.has(checked.actor))
|
|
15758
|
+
return failed();
|
|
15759
|
+
const wait = w.limiter.take(`dlg:${checked.sub}`, d.clock.now().getTime());
|
|
15760
|
+
if (wait > 0)
|
|
15761
|
+
return no(problem(429, "rate_limited", `Too many requests for this person; try again in ${wait} s.`, { "Retry-After": String(wait) }));
|
|
15762
|
+
const userId = usr.links.userOfSubject(checked.sub);
|
|
15763
|
+
const user = userId ? await d.store.getUser(userId) : null;
|
|
15764
|
+
if (!user || !d.admissions.isRegistered(user.id))
|
|
15765
|
+
return failed();
|
|
15766
|
+
const found = await w.recheck(user, w.confirmedAt(user.id));
|
|
15767
|
+
if (found === "not-member")
|
|
15768
|
+
return no(problem(403, "not_member", "That person is no longer a member of this tracker's server, or lost the role it asks for."));
|
|
15769
|
+
if (found === "unknown")
|
|
15770
|
+
return no(problem(503, "membership_unknown", MEMBERSHIP_UNKNOWN2, { "Retry-After": "60" }));
|
|
15771
|
+
return { ok: true, via: { kind: "delegation", delegation: checked }, user };
|
|
15772
|
+
}
|
|
15773
|
+
async function write(writer, user, r, method, body, base) {
|
|
15774
|
+
if (method === "DELETE")
|
|
15775
|
+
return actAnswer(writer, user, r.id, "delete");
|
|
15776
|
+
if (r.kind === "tasks")
|
|
15777
|
+
return createAnswer(writer, user, body, base);
|
|
15778
|
+
if (r.kind === "task")
|
|
15779
|
+
return editAnswer(writer, user, r.id, body, base);
|
|
15780
|
+
if (r.kind === "act") {
|
|
15781
|
+
if (Object.keys(body).length > 0)
|
|
15782
|
+
return problem(400, "unknown_field", `\`${r.action}\` takes no fields: send {}.`);
|
|
15783
|
+
return actAnswer(writer, user, r.id, r.action);
|
|
15784
|
+
}
|
|
15785
|
+
if (r.kind === "run") {
|
|
15786
|
+
if (Object.keys(body).length > 0)
|
|
15787
|
+
return problem(400, "unknown_field", "`run` takes no fields: send {}.");
|
|
15788
|
+
return runNowAnswer(writer, user, r.id);
|
|
15789
|
+
}
|
|
15790
|
+
if (r.kind === "listings")
|
|
15791
|
+
return listingsAnswer(writer, user, r.id, body);
|
|
15792
|
+
if (r.kind === "sources")
|
|
15793
|
+
return sourcesAnswer(writer, user, r.id, body);
|
|
15794
|
+
if (r.kind === "ready") {
|
|
15795
|
+
if (Object.keys(body).length > 0)
|
|
15796
|
+
return problem(400, "unknown_field", "`ready` takes no fields: send {}.");
|
|
15797
|
+
return sourcesReadyAnswer(writer, user, r.id);
|
|
15798
|
+
}
|
|
15799
|
+
return problem(405, "method_not_allowed", "Not a write.");
|
|
15112
15800
|
}
|
|
15113
15801
|
async function handleApi(w, request, path) {
|
|
15114
15802
|
if (request.headers.get("origin") !== null)
|
|
@@ -15124,45 +15812,83 @@ async function handleApi(w, request, path) {
|
|
|
15124
15812
|
const method = request.method;
|
|
15125
15813
|
if (!allowed.split(", ").includes(method))
|
|
15126
15814
|
return json2(problem(405, "method_not_allowed", `Use ${allowed}.`, { Allow: allowed }));
|
|
15815
|
+
if (auth.via.kind === "delegation") {
|
|
15816
|
+
const operation = operationOf(r, method);
|
|
15817
|
+
if (operation === null)
|
|
15818
|
+
return json2(problem(403, "insufficient_scope", "No delegation token can do this."));
|
|
15819
|
+
const needed = `${w.d.usr?.app}:${operation}`;
|
|
15820
|
+
if (!auth.via.delegation.scopes.has(needed)) {
|
|
15821
|
+
return json2(problem(403, "insufficient_scope", `That token does not allow this: it needs ${needed}.`, { "WWW-Authenticate": `${CHALLENGE}, error="insufficient_scope", scope="${needed}"` }));
|
|
15822
|
+
}
|
|
15823
|
+
if (method !== "GET")
|
|
15824
|
+
w.d.log.info(`api: ${operation} for ${user.id} by usr ${auth.via.delegation.actor} (token ${auth.via.delegation.jti})`);
|
|
15825
|
+
}
|
|
15127
15826
|
const writer = { d: w.d, queue: w.queue, reading: w.reading, userId: user.id, arrived: w.arrived };
|
|
15128
15827
|
const base = `${w.base}${API_PREFIX}`;
|
|
15129
|
-
if (
|
|
15130
|
-
|
|
15828
|
+
if (r.kind === "validate") {
|
|
15829
|
+
if (request.headers.get("if-match") !== null)
|
|
15830
|
+
return json2(problem(400, "if_match_unsupported", "If-Match goes only with PATCH or DELETE of a task, or its pause or resume."));
|
|
15831
|
+
const read = await readJson(request);
|
|
15131
15832
|
if (!read.ok)
|
|
15132
15833
|
return json2(read.answer);
|
|
15133
|
-
|
|
15134
|
-
|
|
15135
|
-
|
|
15136
|
-
|
|
15137
|
-
|
|
15138
|
-
|
|
15139
|
-
if (
|
|
15140
|
-
return json2(problem(400, "
|
|
15141
|
-
|
|
15834
|
+
return json2(await validateAnswer(w.d, user, read.body));
|
|
15835
|
+
}
|
|
15836
|
+
if (method === "POST" || method === "PATCH" || method === "DELETE") {
|
|
15837
|
+
const ifMatchRaw = request.headers.get("if-match");
|
|
15838
|
+
if (ifMatchRaw !== null) {
|
|
15839
|
+
const onOneTask = r.kind === "task" && method !== "POST" || r.kind === "act";
|
|
15840
|
+
if (!onOneTask)
|
|
15841
|
+
return json2(problem(400, "if_match_unsupported", "If-Match goes only with PATCH or DELETE of a task, or its pause or resume."));
|
|
15842
|
+
const version = ifMatchVersion(ifMatchRaw);
|
|
15843
|
+
if (version === null)
|
|
15844
|
+
return json2(problem(400, "invalid_if_match", 'If-Match is one task version in quotes, as its ETag gives it: "0123456789abcdef".'));
|
|
15845
|
+
writer.ifMatch = { taskId: r.id.trim(), version };
|
|
15846
|
+
}
|
|
15847
|
+
const key = request.headers.get("idempotency-key");
|
|
15848
|
+
if (key !== null && !isIdempotencyKey(key))
|
|
15849
|
+
return json2(problem(400, "invalid_idempotency_key", "Idempotency-Key is 8 to 128 letters, digits, '-', '_', '.' or ':'; a UUID will do."));
|
|
15850
|
+
let body = {};
|
|
15851
|
+
if (method !== "DELETE") {
|
|
15852
|
+
const read = await readJson(request, r.kind === "listings" ? MAX_LISTINGS_BYTES : r.kind === "sources" ? MAX_SOURCES_BYTES : MAX_API_BYTES);
|
|
15853
|
+
if (!read.ok)
|
|
15854
|
+
return json2(read.answer);
|
|
15855
|
+
body = read.body;
|
|
15142
15856
|
}
|
|
15143
|
-
if (
|
|
15144
|
-
return json2(await
|
|
15145
|
-
|
|
15146
|
-
|
|
15147
|
-
|
|
15148
|
-
|
|
15149
|
-
|
|
15150
|
-
return json2(
|
|
15857
|
+
if (key === null)
|
|
15858
|
+
return json2(await write(writer, user, r, method, body, base));
|
|
15859
|
+
const hash = sha256(`${method} ${path}
|
|
15860
|
+
${writer.ifMatch?.version ?? ""}
|
|
15861
|
+
${canonicalJson(body)}`);
|
|
15862
|
+
const begun = w.d.apiIdempotency.begin(user.id, key, hash, w.d.clock.now());
|
|
15863
|
+
if (begun.kind === "conflict")
|
|
15864
|
+
return json2(problem(409, "idempotency_conflict", "That Idempotency-Key was used for a different request; use a new key for a new request."));
|
|
15865
|
+
if (begun.kind === "in_progress")
|
|
15866
|
+
return json2(problem(409, "idempotency_in_progress", "A request with that Idempotency-Key is still running; try again in a moment.", { "Retry-After": "1" }));
|
|
15867
|
+
if (begun.kind === "replay")
|
|
15868
|
+
return json2({ status: begun.answer.status, body: begun.answer.body, headers: { ...begun.answer.headers, "Idempotency-Replayed": "true" } });
|
|
15869
|
+
let answer;
|
|
15870
|
+
try {
|
|
15871
|
+
answer = await write(writer, user, r, method, body, base);
|
|
15872
|
+
} catch (err) {
|
|
15873
|
+
w.d.apiIdempotency.fail(user.id, key, begun.claim);
|
|
15874
|
+
throw err;
|
|
15151
15875
|
}
|
|
15876
|
+
w.d.apiIdempotency.finish(user.id, key, begun.claim, { status: answer.status, body: answer.body, headers: answer.headers ?? {} });
|
|
15877
|
+
return json2(answer);
|
|
15152
15878
|
}
|
|
15153
|
-
if (method === "DELETE" && r.kind === "task")
|
|
15154
|
-
return json2(await actAnswer(writer, user, r.id, "delete"));
|
|
15155
15879
|
if (r.kind === "types")
|
|
15156
15880
|
return json2(typesAnswer(w.d));
|
|
15157
15881
|
if (r.kind === "tasks")
|
|
15158
|
-
return json2(await listAnswer(w.d, user));
|
|
15882
|
+
return json2(await listAnswer(w.d, user, new URL(request.url).searchParams));
|
|
15883
|
+
if (r.kind === "capabilities")
|
|
15884
|
+
return json2(capabilitiesAnswer(w.d, user, { burst: RATE_BURST, perSecond: RATE_PER_SECOND }));
|
|
15159
15885
|
if (r.kind === "task")
|
|
15160
15886
|
return json2(await getAnswer(w.d, user, r.id));
|
|
15161
15887
|
return json2({
|
|
15162
15888
|
status: 200,
|
|
15163
15889
|
body: {
|
|
15164
15890
|
user: { id: user.id, name: user.displayName, timeZone: user.timeZone, preferredHour: user.preferredHour },
|
|
15165
|
-
token: { id: auth.token.id, name: auth.token.name, createdAt: auth.token.createdAt, expiresAt: auth.token.expiresAt }
|
|
15891
|
+
...auth.via.kind === "token" ? { token: { id: auth.via.token.id, name: auth.via.token.name, createdAt: auth.via.token.createdAt, expiresAt: auth.via.token.expiresAt } } : { delegation: { actor: auth.via.delegation.actor, scopes: [...auth.via.delegation.scopes].sort(), expiresAt: new Date(auth.via.delegation.expiresAt).toISOString() } }
|
|
15166
15892
|
}
|
|
15167
15893
|
});
|
|
15168
15894
|
}
|
|
@@ -15370,6 +16096,9 @@ var JWKS_RETRY_MS = 30 * 1000;
|
|
|
15370
16096
|
var JWKS_TIMEOUT_MS = 5000;
|
|
15371
16097
|
var MAX_TOKEN = 8192;
|
|
15372
16098
|
var MAX_JWKS = 64 * 1024;
|
|
16099
|
+
var MAX_DELEGATION_MS = 30 * 60 * 1000;
|
|
16100
|
+
var DELEGATION_SKEW_MS = 60 * 1000;
|
|
16101
|
+
var JTI = /^[A-Za-z0-9_-]{1,64}$/;
|
|
15373
16102
|
function decode(part) {
|
|
15374
16103
|
try {
|
|
15375
16104
|
return JSON.parse(Buffer.from(part, "base64url").toString("utf8"));
|
|
@@ -15382,38 +16111,70 @@ class UsrVerifier {
|
|
|
15382
16111
|
o;
|
|
15383
16112
|
keys = [];
|
|
15384
16113
|
fetchedAt = Number.NEGATIVE_INFINITY;
|
|
16114
|
+
loadFailed = false;
|
|
15385
16115
|
pending = null;
|
|
15386
16116
|
constructor(o) {
|
|
15387
16117
|
this.o = o;
|
|
15388
16118
|
}
|
|
15389
16119
|
async verify(token) {
|
|
15390
|
-
|
|
16120
|
+
const signed = await this.signed(token);
|
|
16121
|
+
if (typeof signed === "string" || signed.header.typ === "dlg+jwt")
|
|
16122
|
+
return null;
|
|
16123
|
+
const claims = signed.claims;
|
|
16124
|
+
if (claims.iss !== "usr" || claims.act !== undefined)
|
|
16125
|
+
return null;
|
|
16126
|
+
if (typeof claims.exp !== "number" || claims.exp * 1000 <= this.o.now())
|
|
16127
|
+
return null;
|
|
16128
|
+
if (typeof claims.sub !== "string" || claims.sub === "")
|
|
15391
16129
|
return null;
|
|
16130
|
+
const roles = Array.isArray(claims.roles) ? claims.roles.filter((r) => typeof r === "string") : [];
|
|
16131
|
+
return { sub: claims.sub, roles };
|
|
16132
|
+
}
|
|
16133
|
+
async verifyDelegation(token, audience) {
|
|
16134
|
+
const signed = await this.signed(token);
|
|
16135
|
+
if (typeof signed === "string")
|
|
16136
|
+
return signed;
|
|
16137
|
+
if (signed.header.typ !== "dlg+jwt")
|
|
16138
|
+
return "invalid";
|
|
16139
|
+
const c = signed.claims;
|
|
16140
|
+
if (c.iss !== "usr" || c.aud !== audience)
|
|
16141
|
+
return "invalid";
|
|
16142
|
+
if (typeof c.sub !== "string" || c.sub === "" || typeof c.scope !== "string" || typeof c.jti !== "string" || !JTI.test(c.jti))
|
|
16143
|
+
return "invalid";
|
|
16144
|
+
const actor = c.act?.sub;
|
|
16145
|
+
if (typeof actor !== "string" || !/^key:[0-9]+$/.test(actor))
|
|
16146
|
+
return "invalid";
|
|
16147
|
+
if (typeof c.exp !== "number")
|
|
16148
|
+
return "invalid";
|
|
16149
|
+
const expiresAt = c.exp * 1000;
|
|
16150
|
+
const now = this.o.now();
|
|
16151
|
+
if (expiresAt <= now || expiresAt - now > MAX_DELEGATION_MS + DELEGATION_SKEW_MS)
|
|
16152
|
+
return "invalid";
|
|
16153
|
+
return { sub: c.sub, actor, scopes: new Set(c.scope.split(" ").filter((x) => x !== "")), jti: c.jti, expiresAt };
|
|
16154
|
+
}
|
|
16155
|
+
async signed(token) {
|
|
16156
|
+
if (!token || token.length > MAX_TOKEN)
|
|
16157
|
+
return "invalid";
|
|
15392
16158
|
const parts = token.split(".");
|
|
15393
16159
|
if (parts.length !== 3)
|
|
15394
|
-
return
|
|
16160
|
+
return "invalid";
|
|
15395
16161
|
const [h, p, s] = parts;
|
|
15396
16162
|
const header = decode(h);
|
|
15397
|
-
if (!header || header.alg !== "ES256" ||
|
|
15398
|
-
return
|
|
16163
|
+
if (!header || header.alg !== "ES256" || typeof header.kid !== "string")
|
|
16164
|
+
return "invalid";
|
|
15399
16165
|
const key = await this.keyFor(header.kid, false) ?? await this.keyFor(header.kid, true);
|
|
15400
16166
|
if (!key)
|
|
15401
|
-
return
|
|
16167
|
+
return this.loadFailed ? "unavailable" : "invalid";
|
|
15402
16168
|
const signature = Buffer.from(s, "base64url");
|
|
15403
16169
|
if (signature.length !== 64)
|
|
15404
|
-
return
|
|
16170
|
+
return "invalid";
|
|
15405
16171
|
const ok = await crypto.subtle.verify({ name: "ECDSA", hash: "SHA-256" }, key, signature, new TextEncoder().encode(`${h}.${p}`));
|
|
15406
16172
|
if (!ok)
|
|
15407
|
-
return
|
|
16173
|
+
return "invalid";
|
|
15408
16174
|
const claims = decode(p);
|
|
15409
|
-
if (
|
|
15410
|
-
return
|
|
15411
|
-
|
|
15412
|
-
return null;
|
|
15413
|
-
if (typeof claims.sub !== "string" || claims.sub === "")
|
|
15414
|
-
return null;
|
|
15415
|
-
const roles = Array.isArray(claims.roles) ? claims.roles.filter((r) => typeof r === "string") : [];
|
|
15416
|
-
return { sub: claims.sub, roles };
|
|
16175
|
+
if (claims === null || typeof claims !== "object")
|
|
16176
|
+
return "invalid";
|
|
16177
|
+
return { header, claims };
|
|
15417
16178
|
}
|
|
15418
16179
|
async keyFor(kid, force) {
|
|
15419
16180
|
const fresh = () => {
|
|
@@ -15440,6 +16201,7 @@ class UsrVerifier {
|
|
|
15440
16201
|
}
|
|
15441
16202
|
async load() {
|
|
15442
16203
|
this.fetchedAt = this.o.now();
|
|
16204
|
+
this.loadFailed = true;
|
|
15443
16205
|
try {
|
|
15444
16206
|
const res = await (this.o.fetchImpl ?? fetch)(`${this.o.url}/.well-known/jwks.json`, {
|
|
15445
16207
|
headers: { Accept: "application/json" },
|
|
@@ -15464,6 +16226,7 @@ class UsrVerifier {
|
|
|
15464
16226
|
} catch {}
|
|
15465
16227
|
}
|
|
15466
16228
|
this.keys = keys;
|
|
16229
|
+
this.loadFailed = false;
|
|
15467
16230
|
} catch {}
|
|
15468
16231
|
}
|
|
15469
16232
|
}
|
|
@@ -15557,6 +16320,7 @@ function createWebHandler(w) {
|
|
|
15557
16320
|
const shared = pending;
|
|
15558
16321
|
return lookupWithin(() => shared, MEMBER_CHECK_TIMEOUT_MS);
|
|
15559
16322
|
}
|
|
16323
|
+
const memberConfirmed = new Map;
|
|
15560
16324
|
async function recheck(d, user, checkedAt) {
|
|
15561
16325
|
if (w.guildIds === null)
|
|
15562
16326
|
return "ok";
|
|
@@ -15576,12 +16340,14 @@ function createWebHandler(w) {
|
|
|
15576
16340
|
failedAt.set(user.id, d.clock.now().getTime());
|
|
15577
16341
|
}
|
|
15578
16342
|
if (found === "not-member") {
|
|
16343
|
+
memberConfirmed.delete(user.id);
|
|
15579
16344
|
d.sessions.deleteForUser(user.id);
|
|
15580
16345
|
d.apiTokens.deleteForUser(user.id);
|
|
15581
16346
|
return "not-member";
|
|
15582
16347
|
}
|
|
15583
16348
|
if (found === "member") {
|
|
15584
16349
|
const at = d.clock.now().toISOString();
|
|
16350
|
+
memberConfirmed.set(user.id, at);
|
|
15585
16351
|
d.sessions.confirmMember(user.id, at);
|
|
15586
16352
|
d.apiTokens.confirmMember(user.id, at);
|
|
15587
16353
|
return "ok";
|
|
@@ -15731,7 +16497,7 @@ function createWebHandler(w) {
|
|
|
15731
16497
|
const d = w.deps();
|
|
15732
16498
|
if (!d)
|
|
15733
16499
|
return json2(problem(503, "starting", "The tracker is starting up; try again in a minute.", { "Retry-After": "60" }));
|
|
15734
|
-
return handleApi({ d, queue: w.queue, reading, base, limiter, failed, arrived, recheck: (user, at) => recheck(d, user, at) }, request, info.path);
|
|
16500
|
+
return handleApi({ d, queue: w.queue, reading, base, limiter, failed, arrived, recheck: (user, at) => recheck(d, user, at), confirmedAt: (userId) => memberConfirmed.get(userId) ?? null }, request, info.path);
|
|
15735
16501
|
}
|
|
15736
16502
|
const method = request.method;
|
|
15737
16503
|
if (method !== "GET" && method !== "POST")
|
|
@@ -15878,7 +16644,7 @@ function createWebHandler(w) {
|
|
|
15878
16644
|
}
|
|
15879
16645
|
|
|
15880
16646
|
// plugins/tracker/src/mail.ts
|
|
15881
|
-
import { randomBytes as
|
|
16647
|
+
import { randomBytes as randomBytes4 } from "crypto";
|
|
15882
16648
|
|
|
15883
16649
|
// plugins/tracker/src/ebay-alert.ts
|
|
15884
16650
|
var MAX_ALERT_LISTINGS = MAX_LISTINGS_PER_POST;
|
|
@@ -16059,7 +16825,7 @@ class MailAddresses {
|
|
|
16059
16825
|
const row = this.db.query("SELECT local FROM mail_addresses WHERE user_id = ?").get(userId);
|
|
16060
16826
|
if (row)
|
|
16061
16827
|
return `${row.local}@${this.domain}`;
|
|
16062
|
-
const local = `m-${
|
|
16828
|
+
const local = `m-${randomBytes4(12).toString("hex")}`;
|
|
16063
16829
|
this.db.query("INSERT INTO mail_addresses (user_id, local, created_at) VALUES (?, ?, ?)").run(userId, local, now.toISOString());
|
|
16064
16830
|
return `${local}@${this.domain}`;
|
|
16065
16831
|
}
|
|
@@ -16507,6 +17273,7 @@ function createPlugin(host, options = {}) {
|
|
|
16507
17273
|
const webOrigin = parseWebUrl(host.env.TRACKER_WEB_URL);
|
|
16508
17274
|
const cityHall = parseCityHallConfig(host.env);
|
|
16509
17275
|
const usrConfig = parseUsrConfig(host.env);
|
|
17276
|
+
const delegates = parseDelegates(host.env, usrConfig !== null);
|
|
16510
17277
|
const mailConfig = parseMailConfig(host.env);
|
|
16511
17278
|
const usrClient = usrConfig ? createUsrClient({ config: usrConfig, ...options.usrFetch ? { fetchImpl: options.usrFetch } : {} }) : null;
|
|
16512
17279
|
const budgetUnlimited = parseBudgetUnlimited(host.env[BUDGET_UNLIMITED_KEY]);
|
|
@@ -16788,6 +17555,8 @@ function createPlugin(host, options = {}) {
|
|
|
16788
17555
|
host.log.warn(`execute lane is off, so /research is unavailable: ${cityHall.missing.join(", ")} not set`);
|
|
16789
17556
|
if (usrConfig)
|
|
16790
17557
|
host.log.info(`usr link on: ${usrConfig.url}, app ${usrConfig.app}`);
|
|
17558
|
+
if (delegates.size > 0)
|
|
17559
|
+
host.log.info(`task API takes usr delegation tokens from ${[...delegates].join(", ")}`);
|
|
16791
17560
|
const openedDeps = {
|
|
16792
17561
|
store: openedStore,
|
|
16793
17562
|
admissions: new Admissions(opened),
|
|
@@ -16802,6 +17571,7 @@ function createPlugin(host, options = {}) {
|
|
|
16802
17571
|
logins: new LoginLinks(opened),
|
|
16803
17572
|
sessions: new Sessions(opened),
|
|
16804
17573
|
apiTokens: new ApiTokens(opened),
|
|
17574
|
+
apiIdempotency: new ApiIdempotency(opened),
|
|
16805
17575
|
roster: new Roster(opened),
|
|
16806
17576
|
ceilings,
|
|
16807
17577
|
budgetUnlimited,
|
|
@@ -16821,7 +17591,8 @@ function createPlugin(host, options = {}) {
|
|
|
16821
17591
|
url: usrConfig.url,
|
|
16822
17592
|
verifier: new UsrVerifier({ url: usrConfig.url, now: () => clock.now().getTime(), ...options.usrFetch ? { fetchImpl: options.usrFetch } : {} }),
|
|
16823
17593
|
app: usrConfig.app,
|
|
16824
|
-
links: new UsrLinks(opened)
|
|
17594
|
+
links: new UsrLinks(opened),
|
|
17595
|
+
delegates
|
|
16825
17596
|
} : null
|
|
16826
17597
|
};
|
|
16827
17598
|
if (guildIds === null)
|