@hatk/hatk 0.0.1-alpha.107 → 0.0.1-alpha.108
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/classifiers.d.ts +28 -9
- package/dist/classifiers.d.ts.map +1 -1
- package/dist/classifiers.js +257 -116
- package/dist/config.d.ts +4 -4
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -2
- package/dist/database/db.d.ts +2 -0
- package/dist/database/db.d.ts.map +1 -1
- package/dist/database/db.js +5 -0
- package/dist/indexer.d.ts.map +1 -1
- package/dist/indexer.js +7 -0
- package/dist/main.js +3 -3
- package/package.json +1 -1
package/dist/classifiers.d.ts
CHANGED
|
@@ -192,16 +192,35 @@ export declare function runScan(opts?: {
|
|
|
192
192
|
since?: string;
|
|
193
193
|
}): Promise<ScanProgress>;
|
|
194
194
|
/**
|
|
195
|
-
* Score new content
|
|
196
|
-
* on. What existed before then is never scored
|
|
197
|
-
*
|
|
195
|
+
* Score new content as it is indexed, from the moment this was first switched
|
|
196
|
+
* on. What existed before then is never scored here — a pass over the existing
|
|
197
|
+
* library is a deliberate, manual scan from /admin.
|
|
198
198
|
*
|
|
199
|
-
*
|
|
200
|
-
*
|
|
201
|
-
*
|
|
202
|
-
* due is left to finish; so is a manual scan.
|
|
199
|
+
* Starts with one catch-up pass over everything new since that moment, which
|
|
200
|
+
* picks up what a restart dropped from the queue; anything already scored is
|
|
201
|
+
* skipped on its fingerprint before it costs a model call.
|
|
203
202
|
*/
|
|
204
|
-
export declare function
|
|
205
|
-
export declare function
|
|
203
|
+
export declare function startScoringOnIndex(): Promise<void>;
|
|
204
|
+
export declare function stopScoringOnIndex(): void;
|
|
205
|
+
/**
|
|
206
|
+
* Called by the indexer with each batch it applied. Queues every new record for
|
|
207
|
+
* the record classifiers that cover its collection, and its author for the
|
|
208
|
+
* account classifiers once the author has settled. Returns immediately: the
|
|
209
|
+
* indexer never waits on a model.
|
|
210
|
+
*/
|
|
211
|
+
export declare function noteIndexed(items: Array<{
|
|
212
|
+
action: 'create' | 'delete';
|
|
213
|
+
collection: string;
|
|
214
|
+
uri: string;
|
|
215
|
+
authorDid: string;
|
|
216
|
+
}>): void;
|
|
217
|
+
/**
|
|
218
|
+
* Called by the indexer when a repo backfill lands. Its rows were written
|
|
219
|
+
* without passing through {@link noteIndexed} — a new account's first posts
|
|
220
|
+
* arrive this way — so queue whatever in it is new.
|
|
221
|
+
*/
|
|
222
|
+
export declare function noteBackfilled(did: string): void;
|
|
223
|
+
/** Items waiting to be scored, for tests and the admin view. */
|
|
224
|
+
export declare function pendingCount(): number;
|
|
206
225
|
export {};
|
|
207
226
|
//# sourceMappingURL=classifiers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"classifiers.d.ts","sourceRoot":"","sources":["../src/classifiers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"classifiers.d.ts","sourceRoot":"","sources":["../src/classifiers.ts"],"names":[],"mappings":"AAoDA,kDAAkD;AAClD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAA;IACjC,YAAY,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,EAAE,CAAA;IAC1D,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,sEAAsE;AACtE,MAAM,WAAW,iBAAiB;IAChC,6EAA6E;IAC7E,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,OAAO,EAAE,iBAAiB,CAAA;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,gFAAgF;IAChF,OAAO,EAAE,SAAS,GAAG,QAAQ,CAAA;IAC7B,gEAAgE;IAChE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB;sDACkD;IAClD,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;IAC9C;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAA;IAC/G;;;;OAIG;IACH,UAAU,EAAE,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAA;IAC/E;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,EAAE,CAAA;IACjG;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC3C,+EAA+E;IAC/E,KAAK,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,CAAA;CAC7D;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,gBAAgB;IA3CvD,gFAAgF;aACvE,SAAS,GAAG,QAAQ;IAC7B,gEAAgE;kBAClD,MAAM,EAAE;IACtB;sDACkD;gBACtC,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC;IAC9C;;;;;;;OAOG;eACQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IAC/G;;;;OAIG;gBACS,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC/E;;;;;;;;;OASG;aACM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,EAAE;IACjG;;;;OAIG;gBACS,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAC3C,+EAA+E;YACvE,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;;EAK7D;AAED,UAAU,gBAAiB,SAAQ,gBAAgB;IACjD,IAAI,EAAE,MAAM,CAAA;CACb;AAID,8EAA8E;AAC9E,wBAAsB,eAAe,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAsB3E;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAE/E;AAED,yDAAyD;AACzD,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC;AAED,uCAAuC;AACvC,wBAAgB,eAAe,IAAI,MAAM,EAAE,CAE1C;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAExE;AAID,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CACjB;AAID,wBAAgB,aAAa,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,IAAI,CAEnE;AAED,wBAAgB,gBAAgB,IAAI,OAAO,CAE1C;AA8CD,8EAA8E;AAC9E,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAOD;;;;;;GAMG;AACH,wBAAsB,OAAO,CAC3B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,EAC7C,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GACxE,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC,CA2ClF;AA8BD,MAAM,WAAW,YAAY;IAC3B,2BAA2B;IAC3B,OAAO,EAAE,MAAM,CAAA;IACf,kFAAkF;IAClF,OAAO,EAAE,MAAM,CAAA;IACf,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAA;IACd,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAoBD,yEAAyE;AACzE,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CASjD;AAqLD,iEAAiE;AACjE,wBAAgB,eAAe,IAAI,YAAY,GAAG,IAAI,CAErD;AAED;;;;;;GAMG;AACH,wBAAsB,OAAO,CAC3B,IAAI,GAAE;IACJ,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,CAAA;CACV,GACL,OAAO,CAAC,YAAY,CAAC,CAsEvB;AA+BD;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAQzD;AAED,wBAAgB,kBAAkB,IAAI,IAAI,CAKzC;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,GAChG,IAAI,CAqBN;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBhD;AA6DD,gEAAgE;AAChE,wBAAgB,YAAY,IAAI,MAAM,CAErC"}
|
package/dist/classifiers.js
CHANGED
|
@@ -43,7 +43,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
43
43
|
import { resolve } from 'node:path';
|
|
44
44
|
import { readdirSync } from 'node:fs';
|
|
45
45
|
import { createHash } from 'node:crypto';
|
|
46
|
-
import { querySQL, runSQL, getSchema, listSchemas, getCursor, setCursor, upsertClassification, getClassificationFingerprints, findOpenReport, insertReport, } from "./database/db.js";
|
|
46
|
+
import { querySQL, runSQL, getSchema, listSchemas, getCursor, setCursor, upsertClassification, getClassificationFingerprints, getClassificationFingerprint, findOpenReport, insertReport, } from "./database/db.js";
|
|
47
47
|
import { log, emit } from "./logger.js";
|
|
48
48
|
export function defineClassifier(module) {
|
|
49
49
|
return { __type: 'classifiers', ...module };
|
|
@@ -240,7 +240,6 @@ const dbCtx = {
|
|
|
240
240
|
query: (sql, params) => querySQL(sql, params),
|
|
241
241
|
run: (sql, ...params) => runSQL(sql, params).then(() => undefined),
|
|
242
242
|
};
|
|
243
|
-
/** Every subject a classifier applies to, as `{uri, did, handle, value}`. */
|
|
244
243
|
const TID_CHARS = '234567abcdefghijklmnopqrstuvwxyz';
|
|
245
244
|
/** When a TID record key was minted, or null if the key is not a TID. */
|
|
246
245
|
export function tidTime(rkey) {
|
|
@@ -307,24 +306,113 @@ async function enumerateSubjects(c, since) {
|
|
|
307
306
|
for (const row of rows) {
|
|
308
307
|
if (since && !isNewSince(row.uri, since))
|
|
309
308
|
continue;
|
|
310
|
-
|
|
311
|
-
for (const col of schema.columns) {
|
|
312
|
-
let v = row[col.name];
|
|
313
|
-
if (v === null || v === undefined)
|
|
314
|
-
continue;
|
|
315
|
-
if (col.isJson && typeof v === 'string') {
|
|
316
|
-
try {
|
|
317
|
-
v = JSON.parse(v);
|
|
318
|
-
}
|
|
319
|
-
catch { }
|
|
320
|
-
}
|
|
321
|
-
value[col.originalName] = v;
|
|
322
|
-
}
|
|
323
|
-
out.push({ uri: row.uri, did: row.did, collection, value });
|
|
309
|
+
out.push({ uri: row.uri, did: row.did, collection, value: rowValue(schema, row) });
|
|
324
310
|
}
|
|
325
311
|
}
|
|
326
312
|
return out;
|
|
327
313
|
}
|
|
314
|
+
/** A table row back in the record's own field names, JSON columns parsed. */
|
|
315
|
+
function rowValue(schema, row) {
|
|
316
|
+
const value = {};
|
|
317
|
+
for (const col of schema.columns) {
|
|
318
|
+
let v = row[col.name];
|
|
319
|
+
if (v === null || v === undefined)
|
|
320
|
+
continue;
|
|
321
|
+
if (col.isJson && typeof v === 'string') {
|
|
322
|
+
try {
|
|
323
|
+
v = JSON.parse(v);
|
|
324
|
+
}
|
|
325
|
+
catch { }
|
|
326
|
+
}
|
|
327
|
+
value[col.originalName] = v;
|
|
328
|
+
}
|
|
329
|
+
return value;
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* The questions are part of what produced a score, so a fingerprint covers them
|
|
333
|
+
* as well as the state: rewording a classifier's criteria, or changing which
|
|
334
|
+
* images it sends, invalidates every score written under the old version.
|
|
335
|
+
*/
|
|
336
|
+
function classifierHash(c) {
|
|
337
|
+
return createHash('sha256')
|
|
338
|
+
.update(JSON.stringify(c.questions ?? c.classify?.toString() ?? ''))
|
|
339
|
+
.update(c.images?.toString() ?? '')
|
|
340
|
+
.digest('hex')
|
|
341
|
+
.slice(0, 16);
|
|
342
|
+
}
|
|
343
|
+
/**
|
|
344
|
+
* Score one subject: build its state, skip it if that state was already scored
|
|
345
|
+
* (`known` is the stored fingerprint), otherwise ask the model, store the
|
|
346
|
+
* result, and file a report if a signal crosses its threshold. Throws on a
|
|
347
|
+
* failure, for the caller to count or retry.
|
|
348
|
+
*/
|
|
349
|
+
async function scoreSubject(c, subject, hash, known, signal) {
|
|
350
|
+
const state = await c.buildState({ db: dbCtx, subject });
|
|
351
|
+
if (!state)
|
|
352
|
+
return { outcome: 'skipped' };
|
|
353
|
+
const fingerprint = createHash('sha256').update(hash).update(JSON.stringify(state)).digest('hex').slice(0, 32);
|
|
354
|
+
if (known === fingerprint)
|
|
355
|
+
return { outcome: 'skipped' };
|
|
356
|
+
let answers;
|
|
357
|
+
let model;
|
|
358
|
+
let inputTokens = 0;
|
|
359
|
+
if (c.classify) {
|
|
360
|
+
const raw = await c.classify(state, { db: dbCtx, subject });
|
|
361
|
+
answers = Object.fromEntries(Object.entries(raw).map(([k, v]) => [k, typeof v === 'number' ? { score: v, type: 'local' } : v]));
|
|
362
|
+
model = `local:${c.name}`;
|
|
363
|
+
}
|
|
364
|
+
else {
|
|
365
|
+
const urls = c.images ? (await c.images(state, { db: dbCtx, subject })).slice(0, MAX_IMAGES) : [];
|
|
366
|
+
const images = await Promise.all(urls.map((u) => embedImage(u, signal)));
|
|
367
|
+
const res = await askClef(state, c.questions, { images, signal });
|
|
368
|
+
answers = res.answers;
|
|
369
|
+
model = res.model;
|
|
370
|
+
inputTokens = res.inputTokens;
|
|
371
|
+
}
|
|
372
|
+
let topSignal = null;
|
|
373
|
+
let topScore = 0;
|
|
374
|
+
for (const [name, sig] of Object.entries(answers)) {
|
|
375
|
+
if (sig.score > topScore) {
|
|
376
|
+
topScore = sig.score;
|
|
377
|
+
topSignal = name;
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
await upsertClassification({
|
|
381
|
+
subjectUri: subject.uri,
|
|
382
|
+
subjectDid: subject.did,
|
|
383
|
+
classifier: c.name,
|
|
384
|
+
signals: answers,
|
|
385
|
+
state,
|
|
386
|
+
topSignal,
|
|
387
|
+
topScore,
|
|
388
|
+
fingerprint,
|
|
389
|
+
model,
|
|
390
|
+
});
|
|
391
|
+
// File on the highest-scoring signal that crosses its own threshold — a
|
|
392
|
+
// second report for the same subject would only split the review.
|
|
393
|
+
let crossing = null;
|
|
394
|
+
for (const [name, sig] of Object.entries(answers)) {
|
|
395
|
+
const t = thresholdFor(c, name);
|
|
396
|
+
if (t != null && sig.score >= t && (!crossing || sig.score > crossing.score)) {
|
|
397
|
+
crossing = { signal: name, score: sig.score };
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
let filed = false;
|
|
401
|
+
if (crossing) {
|
|
402
|
+
const label = labelFor(c, crossing.signal, crossing.score);
|
|
403
|
+
if (!(await findOpenReport(subject.uri, label))) {
|
|
404
|
+
await insertReport({
|
|
405
|
+
subjectUri: subject.uri,
|
|
406
|
+
subjectDid: subject.did,
|
|
407
|
+
label,
|
|
408
|
+
reason: `${c.name}: ${crossing.signal} ${crossing.score.toFixed(2)}`,
|
|
409
|
+
reportedBy: `system:${c.name}`,
|
|
410
|
+
});
|
|
411
|
+
filed = true;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
return { outcome: 'scored', filed, inputTokens };
|
|
415
|
+
}
|
|
328
416
|
let current = null;
|
|
329
417
|
/** Progress of the scan in flight, or the last one to finish. */
|
|
330
418
|
export function getScanProgress() {
|
|
@@ -362,15 +450,7 @@ export async function runScan(opts = {}) {
|
|
|
362
450
|
for (const c of selected) {
|
|
363
451
|
const subjects = await enumerateSubjects(c, opts.since);
|
|
364
452
|
const seen = opts.force ? new Map() : await getClassificationFingerprints(c.name);
|
|
365
|
-
|
|
366
|
-
// covers them as well as the state: rewording a classifier's criteria,
|
|
367
|
-
// or changing which images it sends, invalidates every score written
|
|
368
|
-
// under the old version.
|
|
369
|
-
const questionsHash = createHash('sha256')
|
|
370
|
-
.update(JSON.stringify(c.questions ?? c.classify?.toString() ?? ''))
|
|
371
|
-
.update(c.images?.toString() ?? '')
|
|
372
|
-
.digest('hex')
|
|
373
|
-
.slice(0, 16);
|
|
453
|
+
const hash = classifierHash(c);
|
|
374
454
|
const queue = opts.limit ? subjects.slice(0, opts.limit) : subjects;
|
|
375
455
|
const concurrency = opts.concurrency ?? 8;
|
|
376
456
|
let cursor = 0;
|
|
@@ -381,77 +461,15 @@ export async function runScan(opts = {}) {
|
|
|
381
461
|
const subject = queue[cursor++];
|
|
382
462
|
progress.scanned++;
|
|
383
463
|
try {
|
|
384
|
-
const
|
|
385
|
-
if (
|
|
464
|
+
const res = await scoreSubject(c, subject, hash, seen.get(subject.uri), opts.signal);
|
|
465
|
+
if (res.outcome === 'skipped') {
|
|
386
466
|
progress.skipped++;
|
|
387
467
|
continue;
|
|
388
468
|
}
|
|
389
|
-
const fingerprint = createHash('sha256')
|
|
390
|
-
.update(questionsHash)
|
|
391
|
-
.update(JSON.stringify(state))
|
|
392
|
-
.digest('hex')
|
|
393
|
-
.slice(0, 32);
|
|
394
|
-
if (seen.get(subject.uri) === fingerprint) {
|
|
395
|
-
progress.skipped++;
|
|
396
|
-
continue;
|
|
397
|
-
}
|
|
398
|
-
let answers;
|
|
399
|
-
let model;
|
|
400
|
-
if (c.classify) {
|
|
401
|
-
const raw = await c.classify(state, { db: dbCtx, subject });
|
|
402
|
-
answers = Object.fromEntries(Object.entries(raw).map(([k, v]) => [k, typeof v === 'number' ? { score: v, type: 'local' } : v]));
|
|
403
|
-
model = `local:${c.name}`;
|
|
404
|
-
}
|
|
405
|
-
else {
|
|
406
|
-
const urls = c.images ? (await c.images(state, { db: dbCtx, subject })).slice(0, MAX_IMAGES) : [];
|
|
407
|
-
const images = await Promise.all(urls.map((u) => embedImage(u, opts.signal)));
|
|
408
|
-
const res = await askClef(state, c.questions, { images, signal: opts.signal });
|
|
409
|
-
answers = res.answers;
|
|
410
|
-
model = res.model;
|
|
411
|
-
progress.inputTokens += res.inputTokens;
|
|
412
|
-
}
|
|
413
|
-
let topSignal = null;
|
|
414
|
-
let topScore = 0;
|
|
415
|
-
for (const [name, sig] of Object.entries(answers)) {
|
|
416
|
-
if (sig.score > topScore) {
|
|
417
|
-
topScore = sig.score;
|
|
418
|
-
topSignal = name;
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
await upsertClassification({
|
|
422
|
-
subjectUri: subject.uri,
|
|
423
|
-
subjectDid: subject.did,
|
|
424
|
-
classifier: c.name,
|
|
425
|
-
signals: answers,
|
|
426
|
-
state,
|
|
427
|
-
topSignal,
|
|
428
|
-
topScore,
|
|
429
|
-
fingerprint,
|
|
430
|
-
model,
|
|
431
|
-
});
|
|
432
469
|
progress.scored++;
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
for (const [name, sig] of Object.entries(answers)) {
|
|
437
|
-
const t = thresholdFor(c, name);
|
|
438
|
-
if (t != null && sig.score >= t && (!crossing || sig.score > crossing.score)) {
|
|
439
|
-
crossing = { signal: name, score: sig.score };
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
|
-
if (crossing) {
|
|
443
|
-
const label = labelFor(c, crossing.signal, crossing.score);
|
|
444
|
-
if (!(await findOpenReport(subject.uri, label))) {
|
|
445
|
-
await insertReport({
|
|
446
|
-
subjectUri: subject.uri,
|
|
447
|
-
subjectDid: subject.did,
|
|
448
|
-
label,
|
|
449
|
-
reason: `${c.name}: ${crossing.signal} ${crossing.score.toFixed(2)}`,
|
|
450
|
-
reportedBy: `system:${c.name}`,
|
|
451
|
-
});
|
|
452
|
-
progress.filed++;
|
|
453
|
-
}
|
|
454
|
-
}
|
|
470
|
+
progress.inputTokens += res.inputTokens;
|
|
471
|
+
if (res.filed)
|
|
472
|
+
progress.filed++;
|
|
455
473
|
}
|
|
456
474
|
catch (err) {
|
|
457
475
|
if (err?.name === 'AbortError')
|
|
@@ -479,37 +497,160 @@ export async function runScan(opts = {}) {
|
|
|
479
497
|
}
|
|
480
498
|
return progress;
|
|
481
499
|
}
|
|
482
|
-
// ──
|
|
483
|
-
/** Where
|
|
500
|
+
// ── Scoring on index ───────────────────────────────────────────────────────
|
|
501
|
+
/** Where scoring's starting point is kept, so a restart keeps it. */
|
|
484
502
|
const SINCE_CURSOR = 'classifiers:since';
|
|
485
|
-
let schedule = null;
|
|
486
503
|
/**
|
|
487
|
-
*
|
|
488
|
-
*
|
|
489
|
-
*
|
|
504
|
+
* How long an account waits after its last write before it is scored. An import
|
|
505
|
+
* writes hundreds of photos in a row, and an account's state counts them, so
|
|
506
|
+
* scoring on every write would ask the model once per photo; waiting for the
|
|
507
|
+
* account to settle asks once.
|
|
508
|
+
*/
|
|
509
|
+
const ACCOUNT_SETTLE_MS = 60_000;
|
|
510
|
+
const LIVE_CONCURRENCY = 4;
|
|
511
|
+
const MAX_ATTEMPTS = 5;
|
|
512
|
+
let since = null;
|
|
513
|
+
const pending = new Map();
|
|
514
|
+
const settling = new Map();
|
|
515
|
+
let inFlight = 0;
|
|
516
|
+
/**
|
|
517
|
+
* Score new content as it is indexed, from the moment this was first switched
|
|
518
|
+
* on. What existed before then is never scored here — a pass over the existing
|
|
519
|
+
* library is a deliberate, manual scan from /admin.
|
|
490
520
|
*
|
|
491
|
-
*
|
|
492
|
-
*
|
|
493
|
-
*
|
|
494
|
-
* due is left to finish; so is a manual scan.
|
|
521
|
+
* Starts with one catch-up pass over everything new since that moment, which
|
|
522
|
+
* picks up what a restart dropped from the queue; anything already scored is
|
|
523
|
+
* skipped on its fingerprint before it costs a model call.
|
|
495
524
|
*/
|
|
496
|
-
export async function
|
|
497
|
-
|
|
498
|
-
let since = await getCursor(SINCE_CURSOR);
|
|
525
|
+
export async function startScoringOnIndex() {
|
|
526
|
+
since = await getCursor(SINCE_CURSOR);
|
|
499
527
|
if (!since) {
|
|
500
528
|
since = new Date().toISOString();
|
|
501
529
|
await setCursor(SINCE_CURSOR, since);
|
|
502
530
|
}
|
|
503
|
-
log(`[classifiers]
|
|
504
|
-
|
|
505
|
-
|
|
531
|
+
log(`[classifiers] scoring on index, for content new since ${since}`);
|
|
532
|
+
runScan({ since }).catch((err) => emit('classifiers', 'catch_up_error', { error: err.message }));
|
|
533
|
+
}
|
|
534
|
+
export function stopScoringOnIndex() {
|
|
535
|
+
since = null;
|
|
536
|
+
pending.clear();
|
|
537
|
+
for (const t of settling.values())
|
|
538
|
+
clearTimeout(t);
|
|
539
|
+
settling.clear();
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Called by the indexer with each batch it applied. Queues every new record for
|
|
543
|
+
* the record classifiers that cover its collection, and its author for the
|
|
544
|
+
* account classifiers once the author has settled. Returns immediately: the
|
|
545
|
+
* indexer never waits on a model.
|
|
546
|
+
*/
|
|
547
|
+
export function noteIndexed(items) {
|
|
548
|
+
if (!since || !classifiers.length)
|
|
549
|
+
return;
|
|
550
|
+
for (const item of items) {
|
|
551
|
+
if (item.action !== 'create' || !isNewSince(item.uri, since))
|
|
552
|
+
continue;
|
|
553
|
+
for (const c of classifiers) {
|
|
554
|
+
if (c.subject === 'record' && c.collections?.includes(item.collection)) {
|
|
555
|
+
enqueue({ classifier: c, collection: item.collection, uri: item.uri, did: item.authorDid, attempts: 0 });
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
if (!classifiers.some((c) => c.subject === 'account'))
|
|
559
|
+
continue;
|
|
560
|
+
clearTimeout(settling.get(item.authorDid));
|
|
561
|
+
const did = item.authorDid;
|
|
562
|
+
const timer = setTimeout(() => {
|
|
563
|
+
settling.delete(did);
|
|
564
|
+
for (const c of classifiers) {
|
|
565
|
+
if (c.subject === 'account')
|
|
566
|
+
enqueue({ classifier: c, uri: did, did, attempts: 0 });
|
|
567
|
+
}
|
|
568
|
+
}, ACCOUNT_SETTLE_MS);
|
|
569
|
+
timer.unref?.();
|
|
570
|
+
settling.set(did, timer);
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
/**
|
|
574
|
+
* Called by the indexer when a repo backfill lands. Its rows were written
|
|
575
|
+
* without passing through {@link noteIndexed} — a new account's first posts
|
|
576
|
+
* arrive this way — so queue whatever in it is new.
|
|
577
|
+
*/
|
|
578
|
+
export function noteBackfilled(did) {
|
|
579
|
+
if (!since || !classifiers.length)
|
|
580
|
+
return;
|
|
581
|
+
const from = since;
|
|
582
|
+
void (async () => {
|
|
583
|
+
const items = [];
|
|
584
|
+
// Every collection, not only the record classifiers': a write anywhere is
|
|
585
|
+
// what queues the account for the account classifiers.
|
|
586
|
+
for (const schema of listSchemas()) {
|
|
587
|
+
const collection = schema.collection;
|
|
588
|
+
const rows = (await querySQL(`SELECT uri FROM ${schema.tableName} WHERE did = $1 AND space IS NULL`, [did]));
|
|
589
|
+
for (const r of rows)
|
|
590
|
+
if (isNewSince(r.uri, from))
|
|
591
|
+
items.push({ action: 'create', collection, uri: r.uri, authorDid: did });
|
|
592
|
+
}
|
|
593
|
+
noteIndexed(items);
|
|
594
|
+
})().catch((err) => emit('classifiers', 'backfill_queue_error', { did, error: err.message }));
|
|
595
|
+
}
|
|
596
|
+
function enqueue(item) {
|
|
597
|
+
pending.set(`${item.classifier.name}|${item.uri}`, item);
|
|
598
|
+
pump();
|
|
599
|
+
}
|
|
600
|
+
function pump() {
|
|
601
|
+
while (inFlight < LIVE_CONCURRENCY && pending.size) {
|
|
602
|
+
const [key, item] = pending.entries().next().value;
|
|
603
|
+
pending.delete(key);
|
|
604
|
+
inFlight++;
|
|
605
|
+
scorePending(item)
|
|
606
|
+
.catch((err) => {
|
|
607
|
+
item.attempts++;
|
|
608
|
+
if (item.attempts >= MAX_ATTEMPTS || !since) {
|
|
609
|
+
emit('classifiers', 'live_dropped', {
|
|
610
|
+
classifier: item.classifier.name,
|
|
611
|
+
subject: item.uri,
|
|
612
|
+
error: err.message,
|
|
613
|
+
});
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
const retry = setTimeout(() => enqueue(item), 2 ** item.attempts * 5_000);
|
|
617
|
+
retry.unref?.();
|
|
618
|
+
})
|
|
619
|
+
.finally(() => {
|
|
620
|
+
inFlight--;
|
|
621
|
+
pump();
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
/** Read the subject as it stands now, and score it if it is still in scope. */
|
|
626
|
+
async function scorePending(item) {
|
|
627
|
+
const c = item.classifier;
|
|
628
|
+
let subject;
|
|
629
|
+
if (item.collection) {
|
|
630
|
+
const schema = getSchema(item.collection);
|
|
631
|
+
if (!schema)
|
|
632
|
+
return;
|
|
633
|
+
const [row] = (await querySQL(`SELECT t.* FROM ${schema.tableName} t
|
|
634
|
+
JOIN _repos r ON r.did = t.did AND r.status = 'active'
|
|
635
|
+
WHERE t.uri = $1 AND t.space IS NULL`, [item.uri]));
|
|
636
|
+
// Deleted, taken down, or in a space since it was queued.
|
|
637
|
+
if (!row)
|
|
638
|
+
return;
|
|
639
|
+
subject = { uri: row.uri, did: row.did, collection: item.collection, value: rowValue(schema, row) };
|
|
640
|
+
}
|
|
641
|
+
else {
|
|
642
|
+
const [repo] = (await querySQL(`SELECT did, handle FROM _repos WHERE did = $1 AND status = 'active'`, [
|
|
643
|
+
item.did,
|
|
644
|
+
]));
|
|
645
|
+
if (!repo)
|
|
506
646
|
return;
|
|
507
|
-
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
647
|
+
subject = { uri: repo.did, did: repo.did, handle: repo.handle };
|
|
648
|
+
}
|
|
649
|
+
const res = await scoreSubject(c, subject, classifierHash(c), await getClassificationFingerprint(c.name, subject.uri));
|
|
650
|
+
if (res.outcome === 'scored' && res.filed)
|
|
651
|
+
log(`[classifiers] ${c.name} filed a report on ${subject.uri}`);
|
|
652
|
+
}
|
|
653
|
+
/** Items waiting to be scored, for tests and the admin view. */
|
|
654
|
+
export function pendingCount() {
|
|
655
|
+
return pending.size + inFlight;
|
|
515
656
|
}
|
package/dist/config.d.ts
CHANGED
|
@@ -187,11 +187,11 @@ export interface ClefConfig {
|
|
|
187
187
|
/** `clef` or `clef-flash`. Thresholds tuned on one do not carry to the other. */
|
|
188
188
|
model: 'clef' | 'clef-flash';
|
|
189
189
|
/**
|
|
190
|
-
*
|
|
191
|
-
*
|
|
192
|
-
*
|
|
190
|
+
* Score new content as the indexer writes it. Only content new since this was
|
|
191
|
+
* first switched on is scored; the library that existed before then is left
|
|
192
|
+
* to a manual scan.
|
|
193
193
|
*/
|
|
194
|
-
|
|
194
|
+
scoreOnIndex: boolean;
|
|
195
195
|
}
|
|
196
196
|
/** Input type for defineConfig — fields that have defaults are optional. */
|
|
197
197
|
export type HatkConfigInput = Partial<Omit<HatkConfig, 'oauth' | 'backfill' | 'push' | 'cdn' | 'jetstream' | 'spaces' | 'clef'>> & {
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAA;IACrC,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;IACnC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC1C,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,iBAAiB,EAAE,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,sBAAsB,EAAE,CAAA;CAC7C;AAED,MAAM,WAAW,sBAAsB;IACrC,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,wFAAwF;IACxF,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,mEAAmE;IACnE,UAAU,CAAC,EAAE,aAAa,EAAE,CAAA;IAC5B,WAAW,EAAE,OAAO,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,qFAAqF;AACrF,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,EAAE,cAAc,CAAA;IACrB,GAAG,CAAC,EAAE,aAAa,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACb;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,sEAAsE;IACtE,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb;;;;;;OAMG;IACH,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB;;;;;;;OAOG;IACH,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,SAAS,GAAG,IAAI,CAAA;IACrB,cAAc,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,kBAAkB,EAAE,MAAM,EAAE,CAAA;IAC5B,QAAQ,EAAE,cAAc,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,KAAK,EAAE,WAAW,GAAG,IAAI,CAAA;IACzB,IAAI,EAAE,UAAU,GAAG,IAAI,CAAA;IACvB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,MAAM,EAAE,YAAY,GAAG,IAAI,CAAA;IAC3B,IAAI,EAAE,UAAU,GAAG,IAAI,CAAA;CACxB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,iFAAiF;IACjF,KAAK,EAAE,MAAM,GAAG,YAAY,CAAA;IAC5B;;;;OAIG;IACH,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAA;IACrC,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;IACnC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC1C,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,iBAAiB,EAAE,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,sBAAsB,EAAE,CAAA;CAC7C;AAED,MAAM,WAAW,sBAAsB;IACrC,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,wFAAwF;IACxF,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,mEAAmE;IACnE,UAAU,CAAC,EAAE,aAAa,EAAE,CAAA;IAC5B,WAAW,EAAE,OAAO,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,qFAAqF;AACrF,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,EAAE,cAAc,CAAA;IACrB,GAAG,CAAC,EAAE,aAAa,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACb;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,sEAAsE;IACtE,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb;;;;;;OAMG;IACH,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB;;;;;;;OAOG;IACH,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,SAAS,GAAG,IAAI,CAAA;IACrB,cAAc,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,kBAAkB,EAAE,MAAM,EAAE,CAAA;IAC5B,QAAQ,EAAE,cAAc,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,KAAK,EAAE,WAAW,GAAG,IAAI,CAAA;IACzB,IAAI,EAAE,UAAU,GAAG,IAAI,CAAA;IACvB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,MAAM,EAAE,YAAY,GAAG,IAAI,CAAA;IAC3B,IAAI,EAAE,UAAU,GAAG,IAAI,CAAA;CACxB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,iFAAiF;IACjF,KAAK,EAAE,MAAM,GAAG,YAAY,CAAA;IAC5B;;;;OAIG;IACH,YAAY,EAAE,OAAO,CAAA;CACtB;AAED,4EAA4E;AAC5E,MAAM,MAAM,eAAe,GAAG,OAAO,CACnC,IAAI,CAAC,UAAU,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,WAAW,GAAG,QAAQ,GAAG,MAAM,CAAC,CAC1F,GAAG;IACF,GAAG,CAAC,EAAE,SAAS,GAAG,IAAI,CAAA;IACtB,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG;QAAE,OAAO,EAAE,iBAAiB,EAAE,CAAA;KAAE,CAAC,GAAG,IAAI,CAAA;IACxE,QAAQ,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAA;IAClC,IAAI,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;IACxB,SAAS,CAAC,EAAE,eAAe,GAAG,IAAI,CAAA;IAClC,MAAM,CAAC,EAAE,YAAY,GAAG,IAAI,CAAA;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,CAAA;CAClC,CAAA;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,eAAe,CAErE;AAED,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CA+FxE"}
|
package/dist/config.js
CHANGED
|
@@ -75,12 +75,11 @@ export async function loadConfig(configPath) {
|
|
|
75
75
|
const apiToken = env.CLOUDFLARE_AI_TOKEN || raw?.apiToken;
|
|
76
76
|
if (!accountId || !apiToken)
|
|
77
77
|
return null;
|
|
78
|
-
const interval = parseInt(env.CLEF_SCAN_INTERVAL || '') || raw?.scanInterval || null;
|
|
79
78
|
return {
|
|
80
79
|
accountId,
|
|
81
80
|
apiToken,
|
|
82
81
|
model: env.CLEF_MODEL || raw?.model || 'clef',
|
|
83
|
-
|
|
82
|
+
scoreOnIndex: env.CLEF_SCORE_ON_INDEX ? env.CLEF_SCORE_ON_INDEX === 'true' : (raw?.scoreOnIndex ?? false),
|
|
84
83
|
};
|
|
85
84
|
})(),
|
|
86
85
|
spaces: parsed.spaces
|
package/dist/database/db.d.ts
CHANGED
|
@@ -244,6 +244,8 @@ export declare function upsertClassification(row: ClassificationRow): Promise<vo
|
|
|
244
244
|
* skip subjects whose state has not changed since.
|
|
245
245
|
*/
|
|
246
246
|
export declare function getClassificationFingerprints(classifier: string): Promise<Map<string, string>>;
|
|
247
|
+
/** The stored fingerprint for one subject under one classifier, if it was scored. */
|
|
248
|
+
export declare function getClassificationFingerprint(classifier: string, subjectUri: string): Promise<string | undefined>;
|
|
247
249
|
/** An open report already filed against this subject with this label, if any. */
|
|
248
250
|
export declare function findOpenReport(subjectUri: string, label: string): Promise<number | null>;
|
|
249
251
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/database/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,WAAW,EAAkB,MAAM,aAAa,CAAA;AAC9D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAI1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC9C,OAAO,EAAc,KAAK,UAAU,EAAE,MAAM,cAAc,CAAA;AAQ1D,wBAAgB,eAAe,IAAI,YAAY,CAE9C;AACD,wBAAgB,aAAa,IAAI,UAAU,CAE1C;AAED,wBAAgB,aAAa,IAAI,IAAI,CAEpC;AAMD,wBAAsB,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,EAAE,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAcnG;AAMD;;;;;;;;;;;;GAYG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,WAAW,EAAE,EAC3B,aAAa,EAAE,MAAM,EAAE,EACvB,eAAe,GAAE,MAAM,EAAO,GAC7B,OAAO,CAAC,eAAe,EAAE,CAAC,CA2O5B;AAED,UAAU,eAAe;IACvB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAA;IACjC,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAoED,wBAAsB,aAAa,CAAC,YAAY,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA8F3F;AAuDD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGnE;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGvE;AAED,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC1E,OAAO,CAAC,IAAI,CAAC,CA0Bf;AAED,0DAA0D;AAC1D,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEjF;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGpE;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAO9G;AAED,wBAAsB,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAOlF;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG1D;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG5D;AAED;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,4CAA4C,CAAA;AAE7E;;;;;GAKG;AACH,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAUnE;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3D;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGvE;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAE3F;AAED,wBAAsB,kBAAkB,CACtC,IAAI,GAAE;IACJ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,CAAC,CAAC,EAAE,MAAM,CAAA;CACN,GACL,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA8B1C;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAmB3E;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAO3E;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAcvE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAKhE;AAED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAI/D;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CASxF;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAiCrD;AAED,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CAkChC;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAqGf;AAWD,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBjF;AAED;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CA4BhH;AAED,4EAA4E;AAC5E,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CASzG;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAClG,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CACR,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAC,CAC7G,CAqBA;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC5B;AASD,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAwO9E;AAuCD,UAAU,SAAS;IACjB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE,SAAc,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAoG9C;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAiCrE;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAsCzF;AAED,qEAAqE;AACrE,wBAAsB,aAAa,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CASjH;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAC9D,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAmI9C;AAGD,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAEtF;AAED,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/E;AAED,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAClE,OAAO,CAAC,OAAO,YAAY,EAAE,YAAY,CAAC,CAE5C;AAED,wBAAgB,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAErE;AAED,kDAAkD;AAClD,wBAAgB,WAAW,IAAI,WAAW,EAAE,CAE3C;AAED,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CASpG;AAED,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAe9B;AAED,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CASvG;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CA8B7B;AAED,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CASpC;AAED,wBAAsB,eAAe,CACnC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAWxB;AAKD,wBAAgB,cAAc,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,CAI1C;AAED,wBAAsB,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAc5G;AAiBD,wBAAgB,UAAU,CACxB,GAAG,EAAE,GAAG,EACR,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAC3C,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GACvD,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAyHrB;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAGhE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CASpF;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAIlE;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAKtF;AAED,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOxE;AAED,wBAAsB,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAO3E;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGlE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAW9E;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAOvF;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAI/E;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAQ9E;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE;IACzC,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;CACnB,GAAG,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAO1B;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE;IACvC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA8B7C;AAED,wBAAsB,aAAa,CACjC,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,UAAU,GAAG,WAAW,EAChC,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAevD;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,CAK1D;AAID,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC9F,6EAA6E;IAC7E,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;CACrB;AAED,wDAAwD;AACxD,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmBhF;AAED;;;GAGG;AACH,wBAAsB,6BAA6B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAMpG;AAED,iFAAiF;AACjF,wBAAsB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAM9F;AAED;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,4EAA4E;IAC5E,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,UAAU,CAAA;IACtC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAsG1C;AAED,yCAAyC;AACzC,wBAAsB,sBAAsB,IAAI,OAAO,CAAC;IACtD,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACpC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB,CAAC,CA2BD;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,MAAM,EACX,WAAW,EAAE,MAAM,EAAE,EACrB,aAAa,SAAI,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAYhC;AAED,8CAA8C;AAC9C,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAa9G;AAED,mEAAmE;AACnE,wBAAsB,yBAAyB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKrF"}
|
|
1
|
+
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/database/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,WAAW,EAAkB,MAAM,aAAa,CAAA;AAC9D,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAI1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC9C,OAAO,EAAc,KAAK,UAAU,EAAE,MAAM,cAAc,CAAA;AAQ1D,wBAAgB,eAAe,IAAI,YAAY,CAE9C;AACD,wBAAgB,aAAa,IAAI,UAAU,CAE1C;AAED,wBAAgB,aAAa,IAAI,IAAI,CAEpC;AAMD,wBAAsB,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,EAAE,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAcnG;AAMD;;;;;;;;;;;;GAYG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,WAAW,EAAE,EAC3B,aAAa,EAAE,MAAM,EAAE,EACvB,eAAe,GAAE,MAAM,EAAO,GAC7B,OAAO,CAAC,eAAe,EAAE,CAAC,CA2O5B;AAED,UAAU,eAAe;IACvB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAA;IACjC,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAoED,wBAAsB,aAAa,CAAC,YAAY,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA8F3F;AAuDD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGnE;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGvE;AAED,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC1E,OAAO,CAAC,IAAI,CAAC,CA0Bf;AAED,0DAA0D;AAC1D,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEjF;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGpE;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAO9G;AAED,wBAAsB,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAOlF;AAED,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG1D;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG5D;AAED;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,4CAA4C,CAAA;AAE7E;;;;;GAKG;AACH,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAUnE;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3D;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGvE;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAE3F;AAED,wBAAsB,kBAAkB,CACtC,IAAI,GAAE;IACJ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,CAAC,CAAC,EAAE,MAAM,CAAA;CACN,GACL,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA8B1C;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAmB3E;AAED,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAO3E;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAcvE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAKhE;AAED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAI/D;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CASxF;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAiCrD;AAED,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,GAAG,EAAE,CAAA;CAAE,CAkChC;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAqGf;AAWD,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBjF;AAED;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CA4BhH;AAED,4EAA4E;AAC5E,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CASzG;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAClG,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CACR,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAC,CAC7G,CAqBA;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC5B;AASD,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAwO9E;AAuCD,UAAU,SAAS;IACjB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAA;CACvB;AAED,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE,SAAc,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAoG9C;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CAiCrE;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAsCzF;AAED,qEAAqE;AACrE,wBAAsB,aAAa,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CASjH;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAC9D,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAmI9C;AAGD,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAEtF;AAED,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/E;AAED,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAClE,OAAO,CAAC,OAAO,YAAY,EAAE,YAAY,CAAC,CAE5C;AAED,wBAAgB,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAErE;AAED,kDAAkD;AAClD,wBAAgB,WAAW,IAAI,WAAW,EAAE,CAE3C;AAED,wBAAsB,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CASpG;AAED,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAe9B;AAED,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,CASvG;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CA8B7B;AAED,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CASpC;AAED,wBAAsB,eAAe,CACnC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,GAC7C,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAWxB;AAKD,wBAAgB,cAAc,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,CAI1C;AAED,wBAAsB,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAc5G;AAiBD,wBAAgB,UAAU,CACxB,GAAG,EAAE,GAAG,EACR,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAC3C,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GACvD,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAyHrB;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAGhE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CASpF;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAIlE;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAKtF;AAED,wBAAsB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOxE;AAED,wBAAsB,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAO3E;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGlE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAW9E;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAOvF;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAI/E;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAQ9E;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE;IACzC,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;CACnB,GAAG,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAO1B;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE;IACvC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA8B7C;AAED,wBAAsB,aAAa,CACjC,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,UAAU,GAAG,WAAW,EAChC,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAevD;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,CAK1D;AAID,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC9F,6EAA6E;IAC7E,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;CACrB;AAED,wDAAwD;AACxD,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmBhF;AAED;;;GAGG;AACH,wBAAsB,6BAA6B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAMpG;AAED,qFAAqF;AACrF,wBAAsB,4BAA4B,CAChD,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAM7B;AAED,iFAAiF;AACjF,wBAAsB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAM9F;AAED;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,4EAA4E;IAC5E,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,UAAU,CAAA;IACtC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAsG1C;AAED,yCAAyC;AACzC,wBAAsB,sBAAsB,IAAI,OAAO,CAAC;IACtD,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACpC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB,CAAC,CA2BD;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,MAAM,EACX,WAAW,EAAE,MAAM,EAAE,EACrB,aAAa,SAAI,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAYhC;AAED,8CAA8C;AAC9C,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAa9G;AAED,mEAAmE;AACnE,wBAAsB,yBAAyB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKrF"}
|
package/dist/database/db.js
CHANGED
|
@@ -1898,6 +1898,11 @@ export async function getClassificationFingerprints(classifier) {
|
|
|
1898
1898
|
const rows = await all(`SELECT subject_uri, fingerprint FROM _classifications WHERE classifier = $1`, [classifier]);
|
|
1899
1899
|
return new Map(rows.map((r) => [r.subject_uri, r.fingerprint]));
|
|
1900
1900
|
}
|
|
1901
|
+
/** The stored fingerprint for one subject under one classifier, if it was scored. */
|
|
1902
|
+
export async function getClassificationFingerprint(classifier, subjectUri) {
|
|
1903
|
+
const rows = await all(`SELECT fingerprint FROM _classifications WHERE classifier = $1 AND subject_uri = $2`, [classifier, subjectUri]);
|
|
1904
|
+
return rows[0]?.fingerprint;
|
|
1905
|
+
}
|
|
1901
1906
|
/** An open report already filed against this subject with this label, if any. */
|
|
1902
1907
|
export async function findOpenReport(subjectUri, label) {
|
|
1903
1908
|
const rows = await all(`SELECT id FROM _reports WHERE subject_uri = $1 AND label = $2 AND status = 'open' LIMIT 1`, [subjectUri, label]);
|
package/dist/indexer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAkDhD,iFAAiF;AACjF,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACtC,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC7B;AAoMD,6GAA6G;AAC7G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAEzC;AAED,6EAA6E;AAC7E,wBAAgB,UAAU,IAAI,MAAM,GAAG,IAAI,CAE1C;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAStD;AAED,uFAAuF;AACvF,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAErH;AAED,+CAA+C;AAC/C,wBAAgB,yBAAyB,IAAI,IAAI,CAIhD;AAED;;;;;;;;;GASG;AACH,wBAAgB,0BAA0B,IAAI,IAAI,CAMjD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAMpD;AAED;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AA0BD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAM3G;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC,CAoBvD;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAM3C;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAuFjF;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,mEAAmE;IACnE,UAAU,CAAC,EAAE,aAAa,EAAE,CAAA;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,oDAAoD;AACpD,UAAU,WAAY,SAAQ,eAAe;IAC3C,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CA4B3E;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkCxE;AAED,6EAA6E;AAC7E,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,wEAAwE;AACxE,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAE/D;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CAgD/D;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CA4CvG;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAgChH;AAKD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAE3F;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,IAAI,CAmE9D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EACxB,KAAK,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAc,GACrC,IAAI,CA6DN"}
|
package/dist/indexer.js
CHANGED
|
@@ -10,6 +10,7 @@ import { rebuildAllIndexes } from "./database/fts.js";
|
|
|
10
10
|
import { log, emit, timer } from "./logger.js";
|
|
11
11
|
import { runLabelRules } from "./labels.js";
|
|
12
12
|
import { fireOnCommitHooks } from "./hooks.js";
|
|
13
|
+
import { noteIndexed, noteBackfilled } from "./classifiers.js";
|
|
13
14
|
import { getLexiconArray } from "./database/schema.js";
|
|
14
15
|
import { validateRecord } from '@bigmoves/lexicon';
|
|
15
16
|
let buffer = [];
|
|
@@ -121,6 +122,8 @@ async function flushBuffer() {
|
|
|
121
122
|
// Fire on-commit hooks for everything the batch applied, in the order it was
|
|
122
123
|
// applied (async, non-blocking)
|
|
123
124
|
fireOnCommitHooks(applied);
|
|
125
|
+
// Queue what is new for the classifiers (non-blocking: never waits on a model)
|
|
126
|
+
noteIndexed(applied);
|
|
124
127
|
// Aggregate collection counts and unique DIDs for wide event
|
|
125
128
|
const collections = {};
|
|
126
129
|
const dids = new Set();
|
|
@@ -418,6 +421,10 @@ export async function triggerAutoBackfill(did, attempt = 0) {
|
|
|
418
421
|
replayErrors++;
|
|
419
422
|
}
|
|
420
423
|
}
|
|
424
|
+
// The writes above, and the repo export before them, skipped the flush the
|
|
425
|
+
// classifiers listen on. An account's first posts land here, so queue them.
|
|
426
|
+
if (status === 'success')
|
|
427
|
+
noteBackfilled(did);
|
|
421
428
|
// Schedule retry if failed and under maxRetries
|
|
422
429
|
const retryInfo = status === 'error' ? await getRepoRetryInfo(did) : null;
|
|
423
430
|
const currentRetryCount = retryInfo?.retryCount ?? 0;
|
package/dist/main.js
CHANGED
|
@@ -22,7 +22,7 @@ import { initFeeds, listFeeds } from "./feeds.js";
|
|
|
22
22
|
import { initXrpc, listXrpc, configureRelay, configureCdn, configureOAuth, callXrpc } from "./xrpc.js";
|
|
23
23
|
import { initOpengraph } from "./opengraph.js";
|
|
24
24
|
import { initLabels, getLabelDefinitions } from "./labels.js";
|
|
25
|
-
import { initClassifiers, listClassifiers, configureClef,
|
|
25
|
+
import { initClassifiers, listClassifiers, configureClef, startScoringOnIndex } from "./classifiers.js";
|
|
26
26
|
import { auxCursorKey, jetstreamCursorKey, startAuxIndexer, startIndexer, sweepReferences } from "./indexer.js";
|
|
27
27
|
import { startJetstreamIndexer } from "./jetstream.js";
|
|
28
28
|
import { rebuildAllIndexes } from "./database/fts.js";
|
|
@@ -122,8 +122,8 @@ configureClef(config.clef && {
|
|
|
122
122
|
model: config.clef.model,
|
|
123
123
|
endpoint: `https://api.cloudflare.com/client/v4/accounts/${config.clef.accountId}/ai/run/@cf/cloudflare/${config.clef.model}`,
|
|
124
124
|
});
|
|
125
|
-
if (config.clef?.
|
|
126
|
-
await
|
|
125
|
+
if (config.clef?.scoreOnIndex && listClassifiers().length)
|
|
126
|
+
await startScoringOnIndex();
|
|
127
127
|
setPrivateCollections(config.privateCollections);
|
|
128
128
|
registerCoreHandlers(collections, config.oauth);
|
|
129
129
|
configureOAuth(config.oauth);
|