@semiont/jobs 0.6.8 → 0.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -16
- package/dist/index.d.ts +146 -153
- package/dist/index.js +47 -79
- package/dist/index.js.map +1 -1
- package/dist/worker-main.js +106 -149
- package/dist/worker-main.js.map +1 -1
- package/package.json +8 -8
package/dist/worker-main.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { replyChannelsFor, createTomlConfigLoader, JOB_TYPES, didToAgent, baseUrl, STARTUP_FETCH_RETRY, retryWithBackoff, serviceAccountToken, isTransientFetchError, isObject, isString, isNumber, RETRY_RULES, getPrimaryMediaType, isGenerationJobParams, capabilitiesOf, assembleAnnotation, findClaimSpan, busRequest, BusRequestError, textSourceOf, yieldsGeometryOf, decodeRepresentation, GENERATABLE_MEDIA_TYPES, locate, createFragmentSelector, deriveViews, isResourceId, estimateTokens, reconcileSelector, getLocaleEnglishName, cutChunk, chunkText } from '@semiont/core';
|
|
2
2
|
import { archivistContentReads, extractPdfTextLayer, withinByteBudget, MAX_PDF_BYTES } from '@semiont/content';
|
|
3
3
|
import { withSpan, SpanKind, recordJobOutcome, recordAnchorOutcome, recordDetectionCall } from '@semiont/observability';
|
|
4
4
|
import { createInferenceClient, answerLimitsRequests, StructuredReadError } from '@semiont/inference';
|
|
@@ -3766,9 +3766,9 @@ var require_mapOneOrManyArgs = __commonJS({
|
|
|
3766
3766
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3767
3767
|
exports.mapOneOrManyArgs = void 0;
|
|
3768
3768
|
var map_1 = require_map();
|
|
3769
|
-
var
|
|
3769
|
+
var isArray = Array.isArray;
|
|
3770
3770
|
function callOrApply(fn, args) {
|
|
3771
|
-
return
|
|
3771
|
+
return isArray(args) ? fn.apply(void 0, __spreadArray([], __read(args))) : fn(args);
|
|
3772
3772
|
}
|
|
3773
3773
|
function mapOneOrManyArgs(fn) {
|
|
3774
3774
|
return map_1.map(function(args) {
|
|
@@ -3913,14 +3913,14 @@ var require_argsArgArrayOrObject = __commonJS({
|
|
|
3913
3913
|
"../../node_modules/rxjs/dist/cjs/internal/util/argsArgArrayOrObject.js"(exports) {
|
|
3914
3914
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3915
3915
|
exports.argsArgArrayOrObject = void 0;
|
|
3916
|
-
var
|
|
3916
|
+
var isArray = Array.isArray;
|
|
3917
3917
|
var getPrototypeOf = Object.getPrototypeOf;
|
|
3918
3918
|
var objectProto = Object.prototype;
|
|
3919
3919
|
var getKeys = Object.keys;
|
|
3920
3920
|
function argsArgArrayOrObject(args) {
|
|
3921
3921
|
if (args.length === 1) {
|
|
3922
3922
|
var first_1 = args[0];
|
|
3923
|
-
if (
|
|
3923
|
+
if (isArray(first_1)) {
|
|
3924
3924
|
return { args: first_1, keys: null };
|
|
3925
3925
|
}
|
|
3926
3926
|
if (isPOJO(first_1)) {
|
|
@@ -4668,9 +4668,9 @@ var require_argsOrArgArray = __commonJS({
|
|
|
4668
4668
|
"../../node_modules/rxjs/dist/cjs/internal/util/argsOrArgArray.js"(exports) {
|
|
4669
4669
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4670
4670
|
exports.argsOrArgArray = void 0;
|
|
4671
|
-
var
|
|
4671
|
+
var isArray = Array.isArray;
|
|
4672
4672
|
function argsOrArgArray(args) {
|
|
4673
|
-
return args.length === 1 &&
|
|
4673
|
+
return args.length === 1 && isArray(args[0]) ? args[0] : args;
|
|
4674
4674
|
}
|
|
4675
4675
|
exports.argsOrArgArray = argsOrArgArray;
|
|
4676
4676
|
}
|
|
@@ -9208,19 +9208,6 @@ var require_cjs = __commonJS({
|
|
|
9208
9208
|
|
|
9209
9209
|
// src/job-claim-adapter.ts
|
|
9210
9210
|
var import_rxjs = __toESM(require_cjs());
|
|
9211
|
-
function readUnitCursors(raw, completedUnits) {
|
|
9212
|
-
if (!isObject(raw)) return {};
|
|
9213
|
-
const done = new Set(completedUnits);
|
|
9214
|
-
const cursors = {};
|
|
9215
|
-
for (const [unit, value] of Object.entries(raw)) {
|
|
9216
|
-
if (done.has(unit) || !isObject(value)) continue;
|
|
9217
|
-
const { next, size, found, emitted } = value;
|
|
9218
|
-
if (!isNumber(next) || !isNumber(size) || next < 0 || size < 1) continue;
|
|
9219
|
-
if (!isNumber(found) || !isNumber(emitted) || found < 0 || emitted < 0) continue;
|
|
9220
|
-
cursors[unit] = { next, size, found, emitted };
|
|
9221
|
-
}
|
|
9222
|
-
return cursors;
|
|
9223
|
-
}
|
|
9224
9211
|
function createJobClaimAdapter(options) {
|
|
9225
9212
|
const { bus, jobTypes } = options;
|
|
9226
9213
|
const activeJob$ = new import_rxjs.BehaviorSubject(null);
|
|
@@ -9239,9 +9226,9 @@ function createJobClaimAdapter(options) {
|
|
|
9239
9226
|
let activeSince = null;
|
|
9240
9227
|
const iso = (t) => t === null ? null : new Date(t).toISOString();
|
|
9241
9228
|
const claimNext = async () => {
|
|
9242
|
-
let
|
|
9229
|
+
let claimed;
|
|
9243
9230
|
try {
|
|
9244
|
-
|
|
9231
|
+
claimed = await busRequest(bus, "job:claim", { types: jobTypes }, 1e4);
|
|
9245
9232
|
} catch (error) {
|
|
9246
9233
|
if (error instanceof BusRequestError) {
|
|
9247
9234
|
if (error.code === "bus.none-pending") return { declined: true };
|
|
@@ -9249,25 +9236,19 @@ function createJobClaimAdapter(options) {
|
|
|
9249
9236
|
}
|
|
9250
9237
|
return { refused: { code: null, message: error instanceof Error ? error.message : String(error) } };
|
|
9251
9238
|
}
|
|
9252
|
-
|
|
9253
|
-
return { refused: { code: null, message: "claimed record carries no job id or type" } };
|
|
9254
|
-
}
|
|
9255
|
-
const completedUnits = isArray(record.metadata?.completedUnits) ? record.metadata.completedUnits.filter(isString) : [];
|
|
9256
|
-
const unitCursors = readUnitCursors(record.metadata?.unitCursors, completedUnits);
|
|
9257
|
-
const params = record.params ?? {};
|
|
9239
|
+
const { metadata, params } = claimed;
|
|
9258
9240
|
return {
|
|
9259
9241
|
job: {
|
|
9260
|
-
jobId:
|
|
9261
|
-
type:
|
|
9262
|
-
resourceId:
|
|
9263
|
-
userId: record.metadata?.userId ?? "",
|
|
9242
|
+
jobId: metadata.id,
|
|
9243
|
+
type: metadata.type,
|
|
9244
|
+
resourceId: params.resourceId,
|
|
9264
9245
|
params,
|
|
9265
|
-
|
|
9266
|
-
|
|
9267
|
-
|
|
9268
|
-
|
|
9269
|
-
retryCount:
|
|
9270
|
-
maxRetries:
|
|
9246
|
+
// Both appear on the record once an attempt has checkpointed, and a
|
|
9247
|
+
// finished unit has no cursor: the dispatcher drops it at the merge.
|
|
9248
|
+
completedUnits: metadata.completedUnits ?? [],
|
|
9249
|
+
unitCursors: metadata.unitCursors ?? {},
|
|
9250
|
+
retryCount: metadata.retryCount,
|
|
9251
|
+
maxRetries: metadata.maxRetries
|
|
9271
9252
|
}
|
|
9272
9253
|
};
|
|
9273
9254
|
};
|
|
@@ -9384,10 +9365,8 @@ function createJobClaimAdapter(options) {
|
|
|
9384
9365
|
function willRetryAfter(metadata, failureClass) {
|
|
9385
9366
|
return failureClass !== "deterministic" && metadata.retryCount < metadata.maxRetries;
|
|
9386
9367
|
}
|
|
9387
|
-
|
|
9388
|
-
|
|
9389
|
-
return KNOWN_JOB_TYPES.has(value);
|
|
9390
|
-
}
|
|
9368
|
+
|
|
9369
|
+
// src/types.ts
|
|
9391
9370
|
function asJobParams(params) {
|
|
9392
9371
|
if (typeof params.resourceId !== "string") {
|
|
9393
9372
|
throw new Error("Job params are missing a resourceId");
|
|
@@ -9949,7 +9928,8 @@ Example format:
|
|
|
9949
9928
|
/**
|
|
9950
9929
|
* Build a prompt for detecting structural tags
|
|
9951
9930
|
*
|
|
9952
|
-
* @param content - The
|
|
9931
|
+
* @param content - The text content to analyze — a chunk sized by the
|
|
9932
|
+
* caller from derived provider limits; NEVER re-truncated here
|
|
9953
9933
|
* @param category - The specific category to detect
|
|
9954
9934
|
* @param schemaName - Human-readable schema name
|
|
9955
9935
|
* @param schemaDescription - Schema description
|
|
@@ -10377,7 +10357,7 @@ ${piece}
|
|
|
10377
10357
|
}
|
|
10378
10358
|
if (items.length * YIELD_COLLAPSE_BAND < counted) {
|
|
10379
10359
|
throw new YieldCollapseError(
|
|
10380
|
-
`Extraction found ${items.length} entities where a count call reports ~${counted} mentions (band \xD7${YIELD_COLLAPSE_BAND}) on a ${piece.length}-char chunk \u2014 silent yield collapse
|
|
10360
|
+
`Extraction found ${items.length} entities where a count call reports ~${counted} mentions (band \xD7${YIELD_COLLAPSE_BAND}) on a ${piece.length}-char chunk \u2014 silent yield collapse: deterministic \u2014 a same-size retry returns the identical under-report.`,
|
|
10381
10361
|
[...items],
|
|
10382
10362
|
{ found: items.length, counted, pieceChars: piece.length }
|
|
10383
10363
|
);
|
|
@@ -10752,8 +10732,6 @@ function compileTypst(source) {
|
|
|
10752
10732
|
rmSync(dir, { recursive: true, force: true });
|
|
10753
10733
|
}
|
|
10754
10734
|
}
|
|
10755
|
-
|
|
10756
|
-
// src/workers/generation/citation-resolver.ts
|
|
10757
10735
|
var CITATION_TOKEN = /\[\[([^\s[\]/]+)(?:\/([^\s[\]/]+))?\]\]/g;
|
|
10758
10736
|
function collectContextResourceIds(context) {
|
|
10759
10737
|
const ids = /* @__PURE__ */ new Set();
|
|
@@ -10779,7 +10757,7 @@ function resolveCitationTokens(content, validResourceIds, logger2) {
|
|
|
10779
10757
|
const annotationId = match[2];
|
|
10780
10758
|
clean += content.slice(last, match.index).replace(/[ \t]+$/, "");
|
|
10781
10759
|
last = match.index + token.length;
|
|
10782
|
-
if (!validResourceIds.has(citedResourceId)) {
|
|
10760
|
+
if (!isResourceId(citedResourceId) || !validResourceIds.has(citedResourceId)) {
|
|
10783
10761
|
logger2.warn("Citation token references an id absent from the provided context \u2014 dropped", {
|
|
10784
10762
|
resourceId: citedResourceId
|
|
10785
10763
|
});
|
|
@@ -11020,9 +10998,8 @@ async function processCommentJob(content, inferenceClient, params, buildAnnotati
|
|
|
11020
10998
|
found += comments.length;
|
|
11021
10999
|
const fresh = dedupe(comments.map(
|
|
11022
11000
|
(c) => (
|
|
11023
|
-
//
|
|
11024
|
-
//
|
|
11025
|
-
// consumers that do language-aware rendering rely on them.
|
|
11001
|
+
// Format and language go on the body TextualBody: optional in the
|
|
11002
|
+
// schema, but consumers that do language-aware rendering rely on them.
|
|
11026
11003
|
buildAnnotation("commenting", c, [
|
|
11027
11004
|
{ type: "TextualBody", value: c.comment, purpose: "commenting", format: "text/plain", language: bodyLanguage }
|
|
11028
11005
|
])
|
|
@@ -11063,8 +11040,7 @@ async function processAssessmentJob(content, inferenceClient, params, buildAnnot
|
|
|
11063
11040
|
const fresh = dedupe(assessments.map(
|
|
11064
11041
|
(a) => (
|
|
11065
11042
|
// Single-object body with purpose aligned to motivation, matching the
|
|
11066
|
-
//
|
|
11067
|
-
// persisted assessments. Do not switch to an array or to
|
|
11043
|
+
// majority of persisted assessments. Do not switch to an array or to
|
|
11068
11044
|
// purpose='describing' — that loses the "this is an assessment, not
|
|
11069
11045
|
// a description" signal and breaks existing readers that access
|
|
11070
11046
|
// `body.value` directly on the object.
|
|
@@ -11132,11 +11108,11 @@ async function processReferenceJob(content, inferenceClient, params, buildAnnota
|
|
|
11132
11108
|
params.includeDescriptiveReferences ?? false,
|
|
11133
11109
|
logger2,
|
|
11134
11110
|
params.sourceLanguage,
|
|
11135
|
-
// Liveness heartbeat
|
|
11136
|
-
//
|
|
11137
|
-
//
|
|
11138
|
-
//
|
|
11139
|
-
//
|
|
11111
|
+
// Liveness heartbeat: fires at chunk boundaries and every ~15 s while a
|
|
11112
|
+
// call is in flight, so a long single-chunk call is not silent. It
|
|
11113
|
+
// repeats the current position rather than inventing an advance — the
|
|
11114
|
+
// stall watchdog, janitor and client timeout need a signal, not a
|
|
11115
|
+
// monotone.
|
|
11140
11116
|
() => emitTypeProgress(entityTypeName),
|
|
11141
11117
|
(verdict) => {
|
|
11142
11118
|
underReported = {
|
|
@@ -11293,7 +11269,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
|
|
|
11293
11269
|
`Unsupported outputMediaType for generation: ${outputMediaType}. Generation produces ${GENERATABLE_MEDIA_TYPES.join(" or ")}.`
|
|
11294
11270
|
);
|
|
11295
11271
|
}
|
|
11296
|
-
const title = params.title
|
|
11272
|
+
const title = params.title;
|
|
11297
11273
|
const entityTypes = (params.entityTypes ?? []).map(String);
|
|
11298
11274
|
if (outputMediaType === "application/pdf") {
|
|
11299
11275
|
onProgress(5, { code: "generating-resource" });
|
|
@@ -11373,12 +11349,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
|
|
|
11373
11349
|
title,
|
|
11374
11350
|
format: outputMediaType,
|
|
11375
11351
|
citations: citations2,
|
|
11376
|
-
|
|
11377
|
-
kind: "generation",
|
|
11378
|
-
resourceId: "",
|
|
11379
|
-
resourceName: title,
|
|
11380
|
-
truncated: generated2.truncated
|
|
11381
|
-
}
|
|
11352
|
+
truncated: generated2.truncated
|
|
11382
11353
|
};
|
|
11383
11354
|
}
|
|
11384
11355
|
onProgress(5, { code: "generating-resource" });
|
|
@@ -11414,12 +11385,7 @@ async function processGenerationJob(inferenceClient, params, onProgress, logger2
|
|
|
11414
11385
|
title,
|
|
11415
11386
|
format: outputMediaType,
|
|
11416
11387
|
citations,
|
|
11417
|
-
|
|
11418
|
-
kind: "generation",
|
|
11419
|
-
resourceId: "",
|
|
11420
|
-
resourceName: title,
|
|
11421
|
-
truncated: generated.truncated
|
|
11422
|
-
}
|
|
11388
|
+
truncated: generated.truncated
|
|
11423
11389
|
};
|
|
11424
11390
|
}
|
|
11425
11391
|
|
|
@@ -11503,11 +11469,11 @@ var CommitDurabilityError = class extends Error {
|
|
|
11503
11469
|
durability;
|
|
11504
11470
|
name = "CommitDurabilityError";
|
|
11505
11471
|
};
|
|
11506
|
-
async function probeDurability(session, resourceId
|
|
11472
|
+
async function probeDurability(session, resourceId, annotations) {
|
|
11507
11473
|
const last = annotations[annotations.length - 1];
|
|
11508
11474
|
if (!last) return "probe-unreachable";
|
|
11509
11475
|
try {
|
|
11510
|
-
await session.client.browse.annotation(resourceId
|
|
11476
|
+
await session.client.browse.annotation(resourceId, last.id).fresh();
|
|
11511
11477
|
return "probe-confirmed";
|
|
11512
11478
|
} catch (error) {
|
|
11513
11479
|
const code = isObject(error) && isString(error.code) ? error.code : void 0;
|
|
@@ -11564,32 +11530,29 @@ function startWorkerProcess(config) {
|
|
|
11564
11530
|
completedUnitsByJob.delete(job.jobId);
|
|
11565
11531
|
unitCursorsByJob.delete(job.jobId);
|
|
11566
11532
|
const failAnnotationId = referenceIdOf(job);
|
|
11567
|
-
|
|
11568
|
-
|
|
11569
|
-
|
|
11570
|
-
|
|
11571
|
-
|
|
11572
|
-
|
|
11573
|
-
|
|
11574
|
-
|
|
11575
|
-
|
|
11576
|
-
|
|
11577
|
-
|
|
11578
|
-
|
|
11579
|
-
|
|
11580
|
-
|
|
11581
|
-
|
|
11582
|
-
|
|
11583
|
-
|
|
11584
|
-
|
|
11585
|
-
|
|
11586
|
-
|
|
11587
|
-
|
|
11588
|
-
|
|
11589
|
-
|
|
11590
|
-
}).catch(() => {
|
|
11591
|
-
});
|
|
11592
|
-
}
|
|
11533
|
+
emitEvent(session, "job:fail", {
|
|
11534
|
+
resourceId: job.resourceId,
|
|
11535
|
+
jobId: job.jobId,
|
|
11536
|
+
jobType: job.type,
|
|
11537
|
+
...failAnnotationId ? { annotationId: failAnnotationId } : {},
|
|
11538
|
+
error: message,
|
|
11539
|
+
...completedUnits && completedUnits.length > 0 ? { completedUnits } : {},
|
|
11540
|
+
// Where each unfinished unit got to. Absent rather than `{}` when
|
|
11541
|
+
// nothing was reached: an empty object would claim units were
|
|
11542
|
+
// tracked and none progressed.
|
|
11543
|
+
...unitCursors && Object.keys(unitCursors).length > 0 ? { unitCursors } : {},
|
|
11544
|
+
...failureClass !== void 0 ? { failureClass } : {},
|
|
11545
|
+
// What the commit path OBSERVED about durability, when the failure
|
|
11546
|
+
// came from a commit at all. Present only on that path: absent means
|
|
11547
|
+
// the question never arose, never that durability was ruled out.
|
|
11548
|
+
...error instanceof CommitDurabilityError ? { durability: error.durability } : {},
|
|
11549
|
+
// Whether this failure is the END, answered by the same predicate
|
|
11550
|
+
// the queue applies at failJob. Without it a client cannot tell a
|
|
11551
|
+
// recovering run from a dead one: it sees job:fail either way and
|
|
11552
|
+
// would end its stream on a job the queue is about to re-run.
|
|
11553
|
+
willRetry: willRetryAfter(job, failureClass)
|
|
11554
|
+
}).catch(() => {
|
|
11555
|
+
});
|
|
11593
11556
|
adapter.failJob(job.jobId, message);
|
|
11594
11557
|
}).finally(() => {
|
|
11595
11558
|
if (activeCancel?.jobId === job.jobId) activeCancel = null;
|
|
@@ -11623,17 +11586,11 @@ async function handleJob(adapter, config, job, completedUnitsByJob = /* @__PURE_
|
|
|
11623
11586
|
}
|
|
11624
11587
|
async function handleJobInner(adapter, config, job, completedUnitsByJob, signal, unitCursorsByJob = /* @__PURE__ */ new Map()) {
|
|
11625
11588
|
const { session, inferenceClient, generator } = config;
|
|
11626
|
-
const { jobId } = job;
|
|
11627
|
-
if (!isJobType(job.type)) {
|
|
11628
|
-
adapter.failJob(jobId, `Unrecognized job type: ${job.type}`);
|
|
11629
|
-
return;
|
|
11630
|
-
}
|
|
11631
|
-
const jobType = job.type;
|
|
11632
|
-
const resourceId$1 = resourceId(job.resourceId);
|
|
11589
|
+
const { jobId, type: jobType, resourceId } = job;
|
|
11633
11590
|
const annotationId = referenceIdOf(job);
|
|
11634
11591
|
const attempt = job.retryCount + 1;
|
|
11635
11592
|
const lifecycleBase = {
|
|
11636
|
-
resourceId
|
|
11593
|
+
resourceId,
|
|
11637
11594
|
jobId,
|
|
11638
11595
|
jobType,
|
|
11639
11596
|
attempt,
|
|
@@ -11652,22 +11609,22 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11652
11609
|
}
|
|
11653
11610
|
let ready = null;
|
|
11654
11611
|
if (jobType !== "generation") {
|
|
11655
|
-
const descriptor = await session.client.browse.resource(resourceId
|
|
11612
|
+
const descriptor = await session.client.browse.resource(resourceId).fresh();
|
|
11656
11613
|
const mediaType = getPrimaryMediaType(descriptor);
|
|
11657
11614
|
const source = await withSpan(
|
|
11658
11615
|
"detection:prepare",
|
|
11659
|
-
() => prepareDetection(mediaType ?? "", config.contentReads, resourceId
|
|
11660
|
-
{ attrs: { "resource.id": resourceId
|
|
11616
|
+
() => prepareDetection(mediaType ?? "", config.contentReads, resourceId, generator, (rid) => session.client.browse.resourceAnchoredText(rid)),
|
|
11617
|
+
{ attrs: { "resource.id": resourceId, "media.type": mediaType ?? "unknown" } }
|
|
11661
11618
|
);
|
|
11662
11619
|
if ("declined" in source) {
|
|
11663
11620
|
if (source.declined === "not-yet") {
|
|
11664
|
-
throw new Error(`Anchored text not yet derived for resource ${resourceId
|
|
11621
|
+
throw new Error(`Anchored text not yet derived for resource ${resourceId} \u2014 Smelter has not settled; retrying`);
|
|
11665
11622
|
}
|
|
11666
11623
|
if (source.declined === "no-extractor") {
|
|
11667
|
-
throw new DeterministicJobError(`Cannot run ${jobType} on resource ${resourceId
|
|
11624
|
+
throw new DeterministicJobError(`Cannot run ${jobType} on resource ${resourceId}: media type '${mediaType ?? "unknown"}' has no extractable text to analyze`);
|
|
11668
11625
|
}
|
|
11669
11626
|
if (source.declined === "no-map" || source.declined === "unknown") {
|
|
11670
|
-
throw new DeterministicJobError(`Cannot run ${jobType} on resource ${resourceId
|
|
11627
|
+
throw new DeterministicJobError(`Cannot run ${jobType} on resource ${resourceId}: anchored-text consult returned '${source.declined}'`);
|
|
11671
11628
|
}
|
|
11672
11629
|
await emitEvent(session, "job:complete", {
|
|
11673
11630
|
...terminalBase(),
|
|
@@ -11698,7 +11655,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11698
11655
|
};
|
|
11699
11656
|
const unitCursors = /* @__PURE__ */ new Map();
|
|
11700
11657
|
const commitChunk = async (annotations, checkpoint) => {
|
|
11701
|
-
record(await commitAnnotations(session,
|
|
11658
|
+
record(await commitAnnotations(session, resourceId, annotations, jobId));
|
|
11702
11659
|
unitCursors.set(checkpoint.unit, checkpoint.cursor);
|
|
11703
11660
|
unitCursorsByJob.set(job.jobId, Object.fromEntries(unitCursors));
|
|
11704
11661
|
await emitEvent(session, "job:checkpoint", {
|
|
@@ -11717,8 +11674,9 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11717
11674
|
// The durability write, per chunk, awaited; folds into the terminal
|
|
11718
11675
|
// durability evidence like every commit, and carries the unit's cursor.
|
|
11719
11676
|
commitChunk,
|
|
11720
|
-
// …and the other direction: where an earlier attempt left each unit
|
|
11721
|
-
//
|
|
11677
|
+
// …and the other direction: where an earlier attempt left each unit, so
|
|
11678
|
+
// a partway unit resumes at its offset instead of the top. Empty on a
|
|
11679
|
+
// first attempt.
|
|
11722
11680
|
job.unitCursors
|
|
11723
11681
|
);
|
|
11724
11682
|
await emitEvent(session, "job:complete", {
|
|
@@ -11736,8 +11694,8 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11736
11694
|
// The durability write, per chunk, awaited; folds into the terminal
|
|
11737
11695
|
// durability evidence like every commit, and carries the unit's cursor.
|
|
11738
11696
|
commitChunk,
|
|
11739
|
-
// …and the other direction: where an earlier attempt left each unit
|
|
11740
|
-
//
|
|
11697
|
+
// …and the other direction: where an earlier attempt left each unit.
|
|
11698
|
+
// Empty on a first attempt.
|
|
11741
11699
|
job.unitCursors
|
|
11742
11700
|
);
|
|
11743
11701
|
await emitEvent(session, "job:complete", {
|
|
@@ -11755,8 +11713,8 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11755
11713
|
// The durability write, per chunk, awaited; folds into the terminal
|
|
11756
11714
|
// durability evidence like every commit, and carries the unit's cursor.
|
|
11757
11715
|
commitChunk,
|
|
11758
|
-
// …and the other direction: where an earlier attempt left each unit
|
|
11759
|
-
//
|
|
11716
|
+
// …and the other direction: where an earlier attempt left each unit.
|
|
11717
|
+
// Empty on a first attempt.
|
|
11760
11718
|
job.unitCursors
|
|
11761
11719
|
);
|
|
11762
11720
|
await emitEvent(session, "job:complete", {
|
|
@@ -11794,8 +11752,8 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11794
11752
|
// The durability write, per chunk, awaited; folds into the terminal
|
|
11795
11753
|
// durability evidence like every commit, and carries the unit's cursor.
|
|
11796
11754
|
commitChunk,
|
|
11797
|
-
// …and the other direction: where an earlier attempt left each unit
|
|
11798
|
-
//
|
|
11755
|
+
// …and the other direction: where an earlier attempt left each unit.
|
|
11756
|
+
// Empty on a first attempt.
|
|
11799
11757
|
job.unitCursors
|
|
11800
11758
|
);
|
|
11801
11759
|
if (signal?.aborted) {
|
|
@@ -11821,8 +11779,8 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11821
11779
|
// The durability write, per chunk, awaited; folds into the terminal
|
|
11822
11780
|
// durability evidence like every commit, and carries the unit's cursor.
|
|
11823
11781
|
commitChunk,
|
|
11824
|
-
// …and the other direction: where an earlier attempt left each unit
|
|
11825
|
-
//
|
|
11782
|
+
// …and the other direction: where an earlier attempt left each unit.
|
|
11783
|
+
// Empty on a first attempt.
|
|
11826
11784
|
job.unitCursors
|
|
11827
11785
|
);
|
|
11828
11786
|
await emitEvent(session, "job:complete", {
|
|
@@ -11842,7 +11800,6 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11842
11800
|
onProgress,
|
|
11843
11801
|
config.logger
|
|
11844
11802
|
);
|
|
11845
|
-
const genParams = job.params;
|
|
11846
11803
|
const genReferenceId = referenceIdOf(job);
|
|
11847
11804
|
const storageUri = job.params.storageUri;
|
|
11848
11805
|
const expectedExtension = capabilitiesOf(genResult.format)?.extension;
|
|
@@ -11859,11 +11816,11 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11859
11816
|
file: Buffer.from(genResult.content),
|
|
11860
11817
|
format: genResult.format,
|
|
11861
11818
|
storageUri,
|
|
11862
|
-
sourceResourceId: resourceId
|
|
11819
|
+
sourceResourceId: resourceId,
|
|
11863
11820
|
...genReferenceId ? { sourceAnnotationId: genReferenceId } : {},
|
|
11864
|
-
...
|
|
11865
|
-
...
|
|
11866
|
-
...
|
|
11821
|
+
...job.params.prompt ? { generationPrompt: job.params.prompt } : {},
|
|
11822
|
+
...job.params.language ? { language: job.params.language } : {},
|
|
11823
|
+
...job.params.entityTypes && job.params.entityTypes.length > 0 ? { entityTypes: job.params.entityTypes } : {},
|
|
11867
11824
|
generator,
|
|
11868
11825
|
// The resource cites the job it fulfils; who asked for it is derived
|
|
11869
11826
|
// downstream from that job's events, never stated here.
|
|
@@ -11873,12 +11830,12 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11873
11830
|
const { annotation: provenanceRef } = assembleAnnotation(
|
|
11874
11831
|
{
|
|
11875
11832
|
motivation: "linking",
|
|
11876
|
-
target: { source:
|
|
11877
|
-
body: { type: "SpecificResource", source:
|
|
11833
|
+
target: { source: resourceId },
|
|
11834
|
+
body: { type: "SpecificResource", source: newResourceId, purpose: "linking" }
|
|
11878
11835
|
},
|
|
11879
11836
|
generator
|
|
11880
11837
|
);
|
|
11881
|
-
record(await commitAnnotations(session,
|
|
11838
|
+
record(await commitAnnotations(session, resourceId, [provenanceRef], jobId));
|
|
11882
11839
|
}
|
|
11883
11840
|
const citationRefs = [];
|
|
11884
11841
|
if (genResult.format === "application/pdf" && genResult.citations.length > 0) {
|
|
@@ -11903,7 +11860,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11903
11860
|
}
|
|
11904
11861
|
const citationRef = buildPdfAnnotation(
|
|
11905
11862
|
layer,
|
|
11906
|
-
|
|
11863
|
+
newResourceId,
|
|
11907
11864
|
generator,
|
|
11908
11865
|
"linking",
|
|
11909
11866
|
{ exact: layer.text.slice(span.start, span.end), start: span.start, end: span.end },
|
|
@@ -11918,7 +11875,7 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11918
11875
|
{
|
|
11919
11876
|
motivation: "linking",
|
|
11920
11877
|
target: {
|
|
11921
|
-
source:
|
|
11878
|
+
source: newResourceId,
|
|
11922
11879
|
selector: [
|
|
11923
11880
|
{ type: "TextPositionSelector", start: citation.start, end: citation.end },
|
|
11924
11881
|
{ type: "TextQuoteSelector", exact: citation.exact }
|
|
@@ -11931,10 +11888,10 @@ async function handleJobInner(adapter, config, job, completedUnitsByJob, signal,
|
|
|
11931
11888
|
citationRefs.push(citationRef);
|
|
11932
11889
|
}
|
|
11933
11890
|
}
|
|
11934
|
-
record(await commitAnnotations(session,
|
|
11891
|
+
record(await commitAnnotations(session, newResourceId, citationRefs, jobId));
|
|
11935
11892
|
await emitEvent(session, "job:complete", {
|
|
11936
11893
|
...terminalBase(),
|
|
11937
|
-
result: { kind: "generation", resourceId: newResourceId, resourceName: genResult.title, truncated: genResult.
|
|
11894
|
+
result: { kind: "generation", resourceId: newResourceId, resourceName: genResult.title, truncated: genResult.truncated }
|
|
11938
11895
|
});
|
|
11939
11896
|
adapter.completeJob();
|
|
11940
11897
|
} else {
|
|
@@ -11985,26 +11942,25 @@ var WORKER_AWAITED_OPERATIONS = [
|
|
|
11985
11942
|
"job:claim",
|
|
11986
11943
|
"browse:resource-requested",
|
|
11987
11944
|
// Canonical geometry for a geometry-bearing detection: the consult behind
|
|
11988
|
-
// `ConsultAnchoredText
|
|
11989
|
-
//
|
|
11990
|
-
// (.plans/WORKER-ANCHORED-TEXT-CHANNEL.md); the census below now fails the
|
|
11945
|
+
// `ConsultAnchoredText`, answered by the Smelter. Without it every PDF
|
|
11946
|
+
// detection job fails at the transport probe; the census below fails the
|
|
11991
11947
|
// BUILD when this list and the declared awaits drift.
|
|
11992
11948
|
"browse:anchored-text-requested",
|
|
11993
|
-
// Durability acknowledgement for a unit's annotations
|
|
11994
|
-
//
|
|
11995
|
-
//
|
|
11996
|
-
//
|
|
11949
|
+
// Durability acknowledgement for a unit's annotations. The worker AWAITS
|
|
11950
|
+
// this one — a unit may not advance until its annotations are in the event
|
|
11951
|
+
// log — so its replies must be in the narrow channel set or every commit
|
|
11952
|
+
// fails fast with `bus.unsubscribed`.
|
|
11997
11953
|
"mark:commit",
|
|
11998
|
-
// The durability probe for a commit whose acknowledgement never routed
|
|
11999
|
-
//
|
|
12000
|
-
//
|
|
12001
|
-
//
|
|
11954
|
+
// The durability probe for a commit whose acknowledgement never routed.
|
|
11955
|
+
// SINGULAR by design: the annotation LIST channel is the multi-MB fan-out
|
|
11956
|
+
// this narrowing exists to keep out, and a rare error path is no reason to
|
|
11957
|
+
// let it in.
|
|
12002
11958
|
"browse:annotation-requested"
|
|
12003
11959
|
];
|
|
12004
11960
|
var WORKER_CONSUMED_BROADCASTS = [
|
|
12005
11961
|
// The queue announcement the claim adapter races for.
|
|
12006
11962
|
"job:queued",
|
|
12007
|
-
// Cooperative cancellation of the ACTIVE job
|
|
11963
|
+
// Cooperative cancellation of the ACTIVE job: it stops at a unit boundary.
|
|
12008
11964
|
"job:cancel-requested"
|
|
12009
11965
|
];
|
|
12010
11966
|
var WORKER_ANSWERED_OPERATIONS = [
|
|
@@ -12126,8 +12082,9 @@ async function startAgentWorker(opts) {
|
|
|
12126
12082
|
generator,
|
|
12127
12083
|
// Byte reads for decode-path media only. A geometry-bearing type's text
|
|
12128
12084
|
// never comes from bytes here — it is CONSULTED from the Smelter's
|
|
12129
|
-
// canonical anchored text over the bus
|
|
12130
|
-
//
|
|
12085
|
+
// canonical anchored text over the bus, and this worker cannot derive
|
|
12086
|
+
// even by mistake: deriving needs the anchored-text store, which only
|
|
12087
|
+
// the Smelter holds.
|
|
12131
12088
|
contentReads: contentReads2,
|
|
12132
12089
|
logger: logger2
|
|
12133
12090
|
});
|