@semiont/jobs 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fs-job-queue.d.ts +79 -0
- package/dist/fs-job-queue.d.ts.map +1 -0
- package/dist/index.d.ts +20 -632
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +65 -56
- package/dist/index.js.map +1 -1
- package/dist/job-claim-adapter.d.ts +76 -0
- package/dist/job-claim-adapter.d.ts.map +1 -0
- package/dist/job-queue-interface.d.ts +19 -0
- package/dist/job-queue-interface.d.ts.map +1 -0
- package/dist/job-queue-state-unit.d.ts +26 -0
- package/dist/job-queue-state-unit.d.ts.map +1 -0
- package/dist/job-worker.d.ts +67 -0
- package/dist/job-worker.d.ts.map +1 -0
- package/dist/processors.d.ts +41 -0
- package/dist/processors.d.ts.map +1 -0
- package/dist/types.d.ts +319 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/worker-main.d.ts +22 -2
- package/dist/worker-main.d.ts.map +1 -0
- package/dist/worker-main.js +165 -114
- package/dist/worker-main.js.map +1 -1
- package/dist/worker-process.d.ts +47 -0
- package/dist/worker-process.d.ts.map +1 -0
- package/dist/workers/annotation-detection.d.ts +61 -0
- package/dist/workers/annotation-detection.d.ts.map +1 -0
- package/dist/workers/detection/entity-extractor.d.ts +42 -0
- package/dist/workers/detection/entity-extractor.d.ts.map +1 -0
- package/dist/workers/detection/motivation-parsers.d.ts +116 -0
- package/dist/workers/detection/motivation-parsers.d.ts.map +1 -0
- package/dist/workers/detection/motivation-prompts.d.ts +57 -0
- package/dist/workers/detection/motivation-prompts.d.ts.map +1 -0
- package/dist/workers/generation/resource-generation.d.ts +23 -0
- package/dist/workers/generation/resource-generation.d.ts.map +1 -0
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -1,633 +1,21 @@
|
|
|
1
|
-
import { Readable } from 'stream';
|
|
2
|
-
import * as _semiont_core from '@semiont/core';
|
|
3
|
-
import { ResourceId, JobId, UserId, EntityType, AnnotationId, Annotation, GatheredContext, TagSchema, Logger, EventBus, components } from '@semiont/core';
|
|
4
|
-
import { SemiontProject } from '@semiont/core/node';
|
|
5
|
-
import { InferenceClient } from '@semiont/inference';
|
|
6
|
-
|
|
7
1
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* -
|
|
15
|
-
* -
|
|
16
|
-
* -
|
|
17
|
-
* -
|
|
18
|
-
*/
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Job metadata - common to all states
|
|
30
|
-
*/
|
|
31
|
-
interface JobMetadata {
|
|
32
|
-
id: JobId;
|
|
33
|
-
type: JobType;
|
|
34
|
-
userId: UserId;
|
|
35
|
-
userName: string;
|
|
36
|
-
userEmail: string;
|
|
37
|
-
userDomain: string;
|
|
38
|
-
created: string;
|
|
39
|
-
retryCount: number;
|
|
40
|
-
maxRetries: number;
|
|
41
|
-
}
|
|
42
|
-
/**
|
|
43
|
-
* Locale conventions for detection/generation params.
|
|
44
|
-
*
|
|
45
|
-
* Two independent locales flow through these jobs:
|
|
46
|
-
*
|
|
47
|
-
* - `language` — *annotation body* locale. The BCP-47 tag the LLM should
|
|
48
|
-
* write generated body text in (comment text, assessment text, generated
|
|
49
|
-
* resource content, tag category label). Sourced from the user's UI
|
|
50
|
-
* locale. Stamped onto the W3C `TextualBody.language` field.
|
|
51
|
-
*
|
|
52
|
-
* - `sourceLanguage` — *source resource* locale. The BCP-47 tag of the
|
|
53
|
-
* content being analyzed. Sourced from `ResourceDescriptor` (carried as
|
|
54
|
-
* `Representation.language` on the primary representation). Used in
|
|
55
|
-
* prompts so the LLM analyzes non-English source correctly even when
|
|
56
|
-
* the user's UI locale differs.
|
|
57
|
-
*
|
|
58
|
-
* Examples: a German user analyzing an English document → `language='de'`,
|
|
59
|
-
* `sourceLanguage='en'`. An English user detecting entities in a French
|
|
60
|
-
* document → `language='en'` (unused for entity references), `sourceLanguage='fr'`.
|
|
61
|
-
*/
|
|
62
|
-
/**
|
|
63
|
-
* Detection job parameters
|
|
64
|
-
*/
|
|
65
|
-
interface DetectionParams {
|
|
66
|
-
resourceId: ResourceId;
|
|
67
|
-
entityTypes: EntityType[];
|
|
68
|
-
includeDescriptiveReferences?: boolean;
|
|
69
|
-
/** Annotation body locale — see locale conventions above. */
|
|
70
|
-
language?: string;
|
|
71
|
-
/** Source-resource locale — see locale conventions above. */
|
|
72
|
-
sourceLanguage?: string;
|
|
73
|
-
}
|
|
74
|
-
/**
|
|
75
|
-
* Generation job parameters
|
|
76
|
-
*/
|
|
77
|
-
interface GenerationParams {
|
|
78
|
-
referenceId: AnnotationId;
|
|
79
|
-
sourceResourceId: ResourceId;
|
|
80
|
-
sourceResourceName: string;
|
|
81
|
-
annotation: Annotation;
|
|
82
|
-
prompt?: string;
|
|
83
|
-
title?: string;
|
|
84
|
-
entityTypes?: EntityType[];
|
|
85
|
-
/** Annotation body locale — language the *generated resource* is written in. */
|
|
86
|
-
language?: string;
|
|
87
|
-
/**
|
|
88
|
-
* Source-resource locale — language of the resource being referenced.
|
|
89
|
-
* Used in the prompt so the LLM understands the embedded source-context
|
|
90
|
-
* snippet correctly when source ≠ target language.
|
|
91
|
-
*/
|
|
92
|
-
sourceLanguage?: string;
|
|
93
|
-
context?: GatheredContext;
|
|
94
|
-
temperature?: number;
|
|
95
|
-
maxTokens?: number;
|
|
96
|
-
storageUri?: string;
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* Highlight detection job parameters
|
|
100
|
-
*/
|
|
101
|
-
interface HighlightDetectionParams {
|
|
102
|
-
resourceId: ResourceId;
|
|
103
|
-
instructions?: string;
|
|
104
|
-
density?: number;
|
|
105
|
-
/** Source-resource locale — see locale conventions above. */
|
|
106
|
-
sourceLanguage?: string;
|
|
107
|
-
}
|
|
108
|
-
/**
|
|
109
|
-
* Assessment detection job parameters
|
|
110
|
-
*/
|
|
111
|
-
interface AssessmentDetectionParams {
|
|
112
|
-
resourceId: ResourceId;
|
|
113
|
-
instructions?: string;
|
|
114
|
-
tone?: 'analytical' | 'critical' | 'balanced' | 'constructive';
|
|
115
|
-
density?: number;
|
|
116
|
-
/** Annotation body locale — see locale conventions above. */
|
|
117
|
-
language?: string;
|
|
118
|
-
/** Source-resource locale — see locale conventions above. */
|
|
119
|
-
sourceLanguage?: string;
|
|
120
|
-
}
|
|
121
|
-
/**
|
|
122
|
-
* Comment detection job parameters
|
|
123
|
-
*/
|
|
124
|
-
interface CommentDetectionParams {
|
|
125
|
-
resourceId: ResourceId;
|
|
126
|
-
instructions?: string;
|
|
127
|
-
tone?: 'scholarly' | 'explanatory' | 'conversational' | 'technical';
|
|
128
|
-
density?: number;
|
|
129
|
-
/** Annotation body locale — see locale conventions above. */
|
|
130
|
-
language?: string;
|
|
131
|
-
/** Source-resource locale — see locale conventions above. */
|
|
132
|
-
sourceLanguage?: string;
|
|
133
|
-
}
|
|
134
|
-
/**
|
|
135
|
-
* Tag detection job parameters.
|
|
136
|
-
*
|
|
137
|
-
* Carries the *full* `TagSchema` (not just an id). The dispatcher resolves
|
|
138
|
-
* the caller-supplied `schemaId` against the per-KB tag-schema projection
|
|
139
|
-
* at job-creation time and embeds the resolved schema here, keeping the
|
|
140
|
-
* worker independent of the registry.
|
|
141
|
-
*/
|
|
142
|
-
interface TagDetectionParams {
|
|
143
|
-
resourceId: ResourceId;
|
|
144
|
-
schema: TagSchema;
|
|
145
|
-
categories: string[];
|
|
146
|
-
/** Annotation body locale — see locale conventions above. */
|
|
147
|
-
language?: string;
|
|
148
|
-
/** Source-resource locale — see locale conventions above. */
|
|
149
|
-
sourceLanguage?: string;
|
|
150
|
-
}
|
|
151
|
-
/**
|
|
152
|
-
* Detection job progress
|
|
153
|
-
*/
|
|
154
|
-
interface DetectionProgress {
|
|
155
|
-
totalEntityTypes: number;
|
|
156
|
-
processedEntityTypes: number;
|
|
157
|
-
currentEntityType?: string;
|
|
158
|
-
entitiesFound: number;
|
|
159
|
-
entitiesEmitted: number;
|
|
160
|
-
}
|
|
161
|
-
/**
|
|
162
|
-
* Detection job result
|
|
163
|
-
*/
|
|
164
|
-
interface DetectionResult {
|
|
165
|
-
totalFound: number;
|
|
166
|
-
totalEmitted: number;
|
|
167
|
-
errors: number;
|
|
168
|
-
}
|
|
169
|
-
/**
|
|
170
|
-
* Generation job progress
|
|
171
|
-
*/
|
|
172
|
-
interface YieldProgress {
|
|
173
|
-
stage: 'fetching' | 'generating' | 'creating' | 'linking';
|
|
174
|
-
percentage: number;
|
|
175
|
-
message?: string;
|
|
176
|
-
}
|
|
177
|
-
/**
|
|
178
|
-
* Generation job result
|
|
179
|
-
*/
|
|
180
|
-
interface GenerationResult {
|
|
181
|
-
resourceId: ResourceId;
|
|
182
|
-
resourceName: string;
|
|
183
|
-
}
|
|
184
|
-
/**
|
|
185
|
-
* Highlight detection job progress
|
|
186
|
-
*/
|
|
187
|
-
interface HighlightDetectionProgress {
|
|
188
|
-
stage: 'analyzing' | 'creating';
|
|
189
|
-
percentage: number;
|
|
190
|
-
message?: string;
|
|
191
|
-
}
|
|
192
|
-
/**
|
|
193
|
-
* Highlight detection job result
|
|
194
|
-
*/
|
|
195
|
-
interface HighlightDetectionResult {
|
|
196
|
-
highlightsFound: number;
|
|
197
|
-
highlightsCreated: number;
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Assessment detection job progress
|
|
201
|
-
*/
|
|
202
|
-
interface AssessmentDetectionProgress {
|
|
203
|
-
stage: 'analyzing' | 'creating';
|
|
204
|
-
percentage: number;
|
|
205
|
-
message?: string;
|
|
206
|
-
}
|
|
207
|
-
/**
|
|
208
|
-
* Assessment detection job result
|
|
209
|
-
*/
|
|
210
|
-
interface AssessmentDetectionResult {
|
|
211
|
-
assessmentsFound: number;
|
|
212
|
-
assessmentsCreated: number;
|
|
213
|
-
}
|
|
214
|
-
/**
|
|
215
|
-
* Comment detection job progress
|
|
216
|
-
*/
|
|
217
|
-
interface CommentDetectionProgress {
|
|
218
|
-
stage: 'analyzing' | 'creating';
|
|
219
|
-
percentage: number;
|
|
220
|
-
message?: string;
|
|
221
|
-
}
|
|
222
|
-
/**
|
|
223
|
-
* Comment detection job result
|
|
224
|
-
*/
|
|
225
|
-
interface CommentDetectionResult {
|
|
226
|
-
commentsFound: number;
|
|
227
|
-
commentsCreated: number;
|
|
228
|
-
}
|
|
229
|
-
/**
|
|
230
|
-
* Tag detection job progress
|
|
231
|
-
*/
|
|
232
|
-
interface TagDetectionProgress {
|
|
233
|
-
stage: 'analyzing' | 'creating';
|
|
234
|
-
percentage: number;
|
|
235
|
-
currentCategory?: string;
|
|
236
|
-
processedCategories: number;
|
|
237
|
-
totalCategories: number;
|
|
238
|
-
message?: string;
|
|
239
|
-
}
|
|
240
|
-
/**
|
|
241
|
-
* Tag detection job result
|
|
242
|
-
*/
|
|
243
|
-
interface TagDetectionResult {
|
|
244
|
-
tagsFound: number;
|
|
245
|
-
tagsCreated: number;
|
|
246
|
-
byCategory: Record<string, number>;
|
|
247
|
-
}
|
|
248
|
-
/**
|
|
249
|
-
* Pending job - just created, waiting to be picked up
|
|
250
|
-
*/
|
|
251
|
-
interface PendingJob<P> {
|
|
252
|
-
status: 'pending';
|
|
253
|
-
metadata: JobMetadata;
|
|
254
|
-
params: P;
|
|
255
|
-
}
|
|
256
|
-
/**
|
|
257
|
-
* Running job - actively being processed
|
|
258
|
-
*/
|
|
259
|
-
interface RunningJob<P, PG> {
|
|
260
|
-
status: 'running';
|
|
261
|
-
metadata: JobMetadata;
|
|
262
|
-
params: P;
|
|
263
|
-
startedAt: string;
|
|
264
|
-
progress: PG;
|
|
265
|
-
}
|
|
266
|
-
/**
|
|
267
|
-
* Complete job - successfully finished
|
|
268
|
-
*/
|
|
269
|
-
interface CompleteJob<P, R> {
|
|
270
|
-
status: 'complete';
|
|
271
|
-
metadata: JobMetadata;
|
|
272
|
-
params: P;
|
|
273
|
-
startedAt: string;
|
|
274
|
-
completedAt: string;
|
|
275
|
-
result: R;
|
|
276
|
-
}
|
|
277
|
-
/**
|
|
278
|
-
* Failed job - encountered an error
|
|
279
|
-
*/
|
|
280
|
-
interface FailedJob<P> {
|
|
281
|
-
status: 'failed';
|
|
282
|
-
metadata: JobMetadata;
|
|
283
|
-
params: P;
|
|
284
|
-
startedAt?: string;
|
|
285
|
-
completedAt: string;
|
|
286
|
-
error: string;
|
|
287
|
-
}
|
|
288
|
-
/**
|
|
289
|
-
* Cancelled job - stopped by user
|
|
290
|
-
*/
|
|
291
|
-
interface CancelledJob<P> {
|
|
292
|
-
status: 'cancelled';
|
|
293
|
-
metadata: JobMetadata;
|
|
294
|
-
params: P;
|
|
295
|
-
startedAt?: string;
|
|
296
|
-
completedAt: string;
|
|
297
|
-
}
|
|
298
|
-
/**
|
|
299
|
-
* Generic job - discriminated union of all states
|
|
300
|
-
*/
|
|
301
|
-
type Job<P, PG, R> = PendingJob<P> | RunningJob<P, PG> | CompleteJob<P, R> | FailedJob<P> | CancelledJob<P>;
|
|
302
|
-
type DetectionJob = Job<DetectionParams, DetectionProgress, DetectionResult>;
|
|
303
|
-
type GenerationJob = Job<GenerationParams, YieldProgress, GenerationResult>;
|
|
304
|
-
type HighlightDetectionJob = Job<HighlightDetectionParams, HighlightDetectionProgress, HighlightDetectionResult>;
|
|
305
|
-
type AssessmentDetectionJob = Job<AssessmentDetectionParams, AssessmentDetectionProgress, AssessmentDetectionResult>;
|
|
306
|
-
type CommentDetectionJob = Job<CommentDetectionParams, CommentDetectionProgress, CommentDetectionResult>;
|
|
307
|
-
type TagDetectionJob = Job<TagDetectionParams, TagDetectionProgress, TagDetectionResult>;
|
|
308
|
-
/**
|
|
309
|
-
* Discriminated union of all job types
|
|
310
|
-
*/
|
|
311
|
-
type AnyJob = DetectionJob | GenerationJob | HighlightDetectionJob | AssessmentDetectionJob | CommentDetectionJob | TagDetectionJob;
|
|
312
|
-
declare function isPendingJob(job: AnyJob): job is PendingJob<any>;
|
|
313
|
-
declare function isRunningJob(job: AnyJob): job is RunningJob<any, any>;
|
|
314
|
-
declare function isCompleteJob(job: AnyJob): job is CompleteJob<any, any>;
|
|
315
|
-
declare function isFailedJob(job: AnyJob): job is FailedJob<any>;
|
|
316
|
-
declare function isCancelledJob(job: AnyJob): job is CancelledJob<any>;
|
|
317
|
-
interface JobQueryFilters {
|
|
318
|
-
status?: JobStatus;
|
|
319
|
-
type?: JobType;
|
|
320
|
-
userId?: UserId;
|
|
321
|
-
limit?: number;
|
|
322
|
-
offset?: number;
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
interface JobQueue {
|
|
326
|
-
initialize(): Promise<void>;
|
|
327
|
-
destroy(): void;
|
|
328
|
-
createJob(job: AnyJob): Promise<void>;
|
|
329
|
-
getJob(jobId: JobId): Promise<AnyJob | null>;
|
|
330
|
-
updateJob(job: AnyJob, oldStatus?: JobStatus): Promise<void>;
|
|
331
|
-
pollNextPendingJob(predicate?: (job: AnyJob) => boolean): Promise<AnyJob | null>;
|
|
332
|
-
cancelJob(jobId: JobId): Promise<boolean>;
|
|
333
|
-
getStats(): Promise<{
|
|
334
|
-
pending: number;
|
|
335
|
-
running: number;
|
|
336
|
-
complete: number;
|
|
337
|
-
failed: number;
|
|
338
|
-
cancelled: number;
|
|
339
|
-
}>;
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
/**
|
|
343
|
-
* Job Queue Manager
|
|
344
|
-
*
|
|
345
|
-
* Filesystem-based job queue with atomic operations.
|
|
346
|
-
* Jobs are stored in directories by status for easy polling.
|
|
347
|
-
*/
|
|
348
|
-
|
|
349
|
-
declare class FsJobQueue implements JobQueue {
|
|
350
|
-
private eventBus?;
|
|
351
|
-
private jobsDir;
|
|
352
|
-
private logger;
|
|
353
|
-
private pendingQueue;
|
|
354
|
-
private watcher;
|
|
355
|
-
private loadDebounceTimer;
|
|
356
|
-
constructor(project: SemiontProject, logger: Logger, eventBus?: EventBus | undefined);
|
|
357
|
-
/**
|
|
358
|
-
* Initialize job queue directories, load pending jobs, and start fs.watch
|
|
359
|
-
*/
|
|
360
|
-
initialize(): Promise<void>;
|
|
361
|
-
/**
|
|
362
|
-
* Clean up watcher
|
|
363
|
-
*/
|
|
364
|
-
destroy(): void;
|
|
365
|
-
/**
|
|
366
|
-
* Load pending jobs from disk into in-memory queue
|
|
367
|
-
*/
|
|
368
|
-
private loadPendingJobs;
|
|
369
|
-
/**
|
|
370
|
-
* Debounced version of loadPendingJobs — fs.watch can fire rapidly
|
|
371
|
-
*/
|
|
372
|
-
private debouncedLoadPendingJobs;
|
|
373
|
-
/**
|
|
374
|
-
* Create a new job
|
|
375
|
-
*/
|
|
376
|
-
createJob(job: AnyJob): Promise<void>;
|
|
377
|
-
/**
|
|
378
|
-
* Get a job by ID (searches all status directories)
|
|
379
|
-
*/
|
|
380
|
-
getJob(jobId: JobId): Promise<AnyJob | null>;
|
|
381
|
-
/**
|
|
382
|
-
* Update a job (atomic: delete old, write new)
|
|
383
|
-
*/
|
|
384
|
-
updateJob(job: AnyJob, oldStatus?: JobStatus): Promise<void>;
|
|
385
|
-
/**
|
|
386
|
-
* Poll for next pending job (FIFO) from in-memory queue.
|
|
387
|
-
* If a predicate is provided, returns the first matching job (skipping non-matching ones).
|
|
388
|
-
*/
|
|
389
|
-
pollNextPendingJob(predicate?: (job: AnyJob) => boolean): Promise<AnyJob | null>;
|
|
390
|
-
/**
|
|
391
|
-
* List jobs with filters
|
|
392
|
-
*/
|
|
393
|
-
listJobs(filters?: JobQueryFilters): Promise<AnyJob[]>;
|
|
394
|
-
/**
|
|
395
|
-
* Cancel a job
|
|
396
|
-
*/
|
|
397
|
-
cancelJob(jobId: JobId): Promise<boolean>;
|
|
398
|
-
/**
|
|
399
|
-
* Clean up old completed/failed jobs (older than retention period)
|
|
400
|
-
*/
|
|
401
|
-
cleanupOldJobs(retentionHours?: number): Promise<number>;
|
|
402
|
-
/**
|
|
403
|
-
* Get job file path
|
|
404
|
-
*/
|
|
405
|
-
private getJobPath;
|
|
406
|
-
/**
|
|
407
|
-
* Get statistics about the queue
|
|
408
|
-
*/
|
|
409
|
-
getStats(): Promise<{
|
|
410
|
-
pending: number;
|
|
411
|
-
running: number;
|
|
412
|
-
complete: number;
|
|
413
|
-
failed: number;
|
|
414
|
-
cancelled: number;
|
|
415
|
-
}>;
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
/**
|
|
419
|
-
* Job Worker Base Class
|
|
420
|
-
*
|
|
421
|
-
* Abstract worker that polls the job queue and processes jobs.
|
|
422
|
-
* Subclasses implement specific job processing logic.
|
|
423
|
-
*/
|
|
424
|
-
|
|
425
|
-
declare abstract class JobWorker {
|
|
426
|
-
private running;
|
|
427
|
-
private currentJob;
|
|
428
|
-
private pollIntervalMs;
|
|
429
|
-
private errorBackoffMs;
|
|
430
|
-
protected jobQueue: JobQueue;
|
|
431
|
-
protected logger: Logger;
|
|
432
|
-
constructor(jobQueue: JobQueue, pollIntervalMs: number | undefined, errorBackoffMs: number | undefined, logger: Logger);
|
|
433
|
-
/**
|
|
434
|
-
* Start the worker (polls queue in loop)
|
|
435
|
-
*/
|
|
436
|
-
start(): Promise<void>;
|
|
437
|
-
/**
|
|
438
|
-
* Stop the worker (graceful shutdown)
|
|
439
|
-
*/
|
|
440
|
-
stop(): Promise<void>;
|
|
441
|
-
/**
|
|
442
|
-
* Poll for next job to process
|
|
443
|
-
*/
|
|
444
|
-
private pollNextJob;
|
|
445
|
-
/**
|
|
446
|
-
* Process a job (handles state transitions and error handling)
|
|
447
|
-
*/
|
|
448
|
-
private processJob;
|
|
449
|
-
/**
|
|
450
|
-
* Handle job failure (retry or move to failed)
|
|
451
|
-
*/
|
|
452
|
-
protected handleJobFailure(job: AnyJob, error: any): Promise<void>;
|
|
453
|
-
/**
|
|
454
|
-
* Update job progress (best-effort, doesn't throw)
|
|
455
|
-
*/
|
|
456
|
-
protected updateJobProgress(job: AnyJob): Promise<void>;
|
|
457
|
-
/**
|
|
458
|
-
* Sleep utility
|
|
459
|
-
*/
|
|
460
|
-
protected sleep(ms: number): Promise<void>;
|
|
461
|
-
/**
|
|
462
|
-
* Emit completion event (optional hook for subclasses)
|
|
463
|
-
* Override this to emit job-specific completion events (e.g., job.completed)
|
|
464
|
-
*/
|
|
465
|
-
protected emitCompletionEvent(_job: RunningJob<any, any>, _result: any): Promise<void>;
|
|
466
|
-
/**
|
|
467
|
-
* Get worker name (for logging)
|
|
468
|
-
*/
|
|
469
|
-
protected abstract getWorkerName(): string;
|
|
470
|
-
/**
|
|
471
|
-
* Check if this worker can process the given job
|
|
472
|
-
*/
|
|
473
|
-
protected abstract canProcessJob(job: AnyJob): boolean;
|
|
474
|
-
/**
|
|
475
|
-
* Execute the job (job-specific logic)
|
|
476
|
-
* This is where the actual work happens
|
|
477
|
-
* Return the result object (or void for jobs without results)
|
|
478
|
-
* Throw an error to trigger retry logic
|
|
479
|
-
*/
|
|
480
|
-
protected abstract executeJob(job: AnyJob): Promise<any>;
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
type Agent = components['schemas']['Agent'];
|
|
484
|
-
/**
|
|
485
|
-
* Progress callback. The three positional args satisfy the minimum
|
|
486
|
-
* `JobProgress` required fields (`percentage`, `message`, `stage`).
|
|
487
|
-
* The fourth optional arg carries job-type-specific fields
|
|
488
|
-
* (`currentEntityType`, `completedEntityTypes`, `requestParams`, etc.)
|
|
489
|
-
* that the progress UI renders.
|
|
490
|
-
*/
|
|
491
|
-
type OnProgress = (percentage: number, message: string, stage: string, extra?: Partial<JobProgress>) => void;
|
|
492
|
-
type JobProgress = components['schemas']['JobProgress'];
|
|
493
|
-
interface ProcessorResult<R> {
|
|
494
|
-
annotations: Record<string, unknown>[];
|
|
495
|
-
result: R;
|
|
496
|
-
}
|
|
497
|
-
declare function processHighlightJob(content: string, inferenceClient: InferenceClient, params: HighlightDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<HighlightDetectionResult>>;
|
|
498
|
-
declare function processCommentJob(content: string, inferenceClient: InferenceClient, params: CommentDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<CommentDetectionResult>>;
|
|
499
|
-
declare function processAssessmentJob(content: string, inferenceClient: InferenceClient, params: AssessmentDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<AssessmentDetectionResult>>;
|
|
500
|
-
declare function processReferenceJob(content: string, inferenceClient: InferenceClient, params: DetectionParams, userId: string, generator: Agent, onProgress: OnProgress, logger?: _semiont_core.Logger): Promise<ProcessorResult<DetectionResult>>;
|
|
501
|
-
declare function processTagJob(content: string, inferenceClient: InferenceClient, params: TagDetectionParams, userId: string, generator: Agent, onProgress: OnProgress): Promise<ProcessorResult<TagDetectionResult>>;
|
|
502
|
-
declare function processGenerationJob(inferenceClient: InferenceClient, params: GenerationParams, onProgress: OnProgress): Promise<{
|
|
503
|
-
content: string;
|
|
504
|
-
title: string;
|
|
505
|
-
format: string;
|
|
506
|
-
result: GenerationResult;
|
|
507
|
-
}>;
|
|
508
|
-
|
|
509
|
-
/**
|
|
510
|
-
* Represents a detected comment with validated position
|
|
511
|
-
*/
|
|
512
|
-
interface CommentMatch {
|
|
513
|
-
exact: string;
|
|
514
|
-
start: number;
|
|
515
|
-
end: number;
|
|
516
|
-
prefix?: string;
|
|
517
|
-
suffix?: string;
|
|
518
|
-
comment: string;
|
|
519
|
-
}
|
|
520
|
-
/**
|
|
521
|
-
* Represents a detected highlight with validated position
|
|
522
|
-
*/
|
|
523
|
-
interface HighlightMatch {
|
|
524
|
-
exact: string;
|
|
525
|
-
start: number;
|
|
526
|
-
end: number;
|
|
527
|
-
prefix?: string;
|
|
528
|
-
suffix?: string;
|
|
529
|
-
}
|
|
530
|
-
/**
|
|
531
|
-
* Represents a detected assessment with validated position
|
|
532
|
-
*/
|
|
533
|
-
interface AssessmentMatch {
|
|
534
|
-
exact: string;
|
|
535
|
-
start: number;
|
|
536
|
-
end: number;
|
|
537
|
-
prefix?: string;
|
|
538
|
-
suffix?: string;
|
|
539
|
-
assessment: string;
|
|
540
|
-
}
|
|
541
|
-
/**
|
|
542
|
-
* Represents a detected tag with validated position
|
|
543
|
-
*/
|
|
544
|
-
interface TagMatch {
|
|
545
|
-
exact: string;
|
|
546
|
-
start: number;
|
|
547
|
-
end: number;
|
|
548
|
-
prefix?: string;
|
|
549
|
-
suffix?: string;
|
|
550
|
-
category: string;
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
/**
|
|
554
|
-
* Annotation Detection
|
|
555
|
-
*
|
|
556
|
-
* Orchestrates the full annotation detection pipeline:
|
|
557
|
-
* 1. Build AI prompts using MotivationPrompts
|
|
558
|
-
* 2. Call AI inference
|
|
559
|
-
* 3. Parse and validate results using MotivationParsers
|
|
560
|
-
*
|
|
561
|
-
* All methods take content as a string parameter.
|
|
562
|
-
* Workers are responsible for fetching content via ContentFetcher.
|
|
563
|
-
*/
|
|
564
|
-
|
|
565
|
-
declare class AnnotationDetection {
|
|
566
|
-
/**
|
|
567
|
-
* Fetch content from a ContentFetcher and read the stream to a string.
|
|
568
|
-
* Shared helper for all workers.
|
|
569
|
-
*/
|
|
570
|
-
static fetchContent(contentFetcher: ContentFetcher, resourceId: ResourceId): Promise<string>;
|
|
571
|
-
/**
|
|
572
|
-
* Detect comments in content.
|
|
573
|
-
*
|
|
574
|
-
* `language` is the locale the LLM should write comment text in (annotation
|
|
575
|
-
* body locale). `sourceLanguage` is the locale of the content being analyzed
|
|
576
|
-
* (source-resource locale). See `types.ts` "Locale conventions" for the
|
|
577
|
-
* full discussion.
|
|
578
|
-
*/
|
|
579
|
-
static detectComments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string): Promise<CommentMatch[]>;
|
|
580
|
-
/**
|
|
581
|
-
* Detect highlights in content.
|
|
582
|
-
*
|
|
583
|
-
* Highlights have no body — only `sourceLanguage` (source-resource locale)
|
|
584
|
-
* applies, used in the prompt so the LLM analyzes non-English source
|
|
585
|
-
* correctly.
|
|
586
|
-
*/
|
|
587
|
-
static detectHighlights(content: string, client: InferenceClient, instructions?: string, density?: number, sourceLanguage?: string): Promise<HighlightMatch[]>;
|
|
588
|
-
/**
|
|
589
|
-
* Detect assessments in content.
|
|
590
|
-
*
|
|
591
|
-
* `language` is the locale the LLM should write assessment text in
|
|
592
|
-
* (annotation body locale). `sourceLanguage` is the locale of the content
|
|
593
|
-
* being analyzed (source-resource locale).
|
|
594
|
-
*/
|
|
595
|
-
static detectAssessments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string): Promise<AssessmentMatch[]>;
|
|
596
|
-
/**
|
|
597
|
-
* Detect tags in content for a specific category.
|
|
598
|
-
*
|
|
599
|
-
* The full `TagSchema` is supplied by the dispatcher (resolved against
|
|
600
|
-
* the per-KB tag-schema projection at job-creation time) so the worker
|
|
601
|
-
* is independent of the registry.
|
|
602
|
-
*
|
|
603
|
-
* `sourceLanguage` is the locale of the content being analyzed. Body-locale
|
|
604
|
-
* (`language`) doesn't influence the tag prompt — categories are schema
|
|
605
|
-
* identifiers, not LLM-generated text — so it's consumed at the body-stamp
|
|
606
|
-
* site, not here.
|
|
607
|
-
*/
|
|
608
|
-
static detectTags(content: string, client: InferenceClient, schema: TagSchema, category: string, sourceLanguage?: string): Promise<TagMatch[]>;
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
/**
|
|
612
|
-
* Resource Generation
|
|
613
|
-
*
|
|
614
|
-
* Generates markdown resources from topics using AI inference.
|
|
615
|
-
*/
|
|
616
|
-
|
|
617
|
-
/**
|
|
618
|
-
* Generate resource content using inference.
|
|
619
|
-
*
|
|
620
|
-
* Locale parameters: `locale` is the *body* locale — the language the
|
|
621
|
-
* generated resource should be written in (sourced from the user's UI
|
|
622
|
-
* locale). `sourceLanguage` is the *source* locale — the language of the
|
|
623
|
-
* referenced resource whose context (selected passage, surrounding text)
|
|
624
|
-
* is embedded into the prompt. They're independent: a German user can
|
|
625
|
-
* generate German content from an English source resource. See
|
|
626
|
-
* `types.ts` "Locale conventions" for the full discussion.
|
|
627
|
-
*/
|
|
628
|
-
declare function generateResourceFromTopic(topic: string, entityTypes: string[], client: InferenceClient, userPrompt?: string, locale?: string, context?: GatheredContext, temperature?: number, maxTokens?: number, logger?: Logger, sourceLanguage?: string): Promise<{
|
|
629
|
-
title: string;
|
|
630
|
-
content: string;
|
|
631
|
-
}>;
|
|
632
|
-
|
|
633
|
-
export { AnnotationDetection, type AnyJob, type AssessmentDetectionJob, type AssessmentDetectionParams, type AssessmentDetectionProgress, type AssessmentDetectionResult, type CancelledJob, type CommentDetectionJob, type CommentDetectionParams, type CommentDetectionProgress, type CommentDetectionResult, type CompleteJob, type ContentFetcher, type DetectionJob, type DetectionParams, type DetectionProgress, type DetectionResult, type FailedJob, FsJobQueue, type GenerationJob, type GenerationParams, type GenerationResult, type HighlightDetectionJob, type HighlightDetectionParams, type HighlightDetectionProgress, type HighlightDetectionResult, type JobMetadata, type JobQueryFilters, type JobQueue, type JobStatus, type JobType, JobWorker, type OnProgress, type PendingJob, type ProcessorResult, type RunningJob, type TagDetectionJob, type TagDetectionParams, type TagDetectionProgress, type TagDetectionResult, type YieldProgress, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
|
|
2
|
+
* @semiont/jobs
|
|
3
|
+
*
|
|
4
|
+
* Job queue and worker infrastructure
|
|
5
|
+
*
|
|
6
|
+
* Provides:
|
|
7
|
+
* - JobQueue interface: backing-store-agnostic contract
|
|
8
|
+
* - FsJobQueue: Filesystem-backed implementation
|
|
9
|
+
* - JobWorker: Abstract base class for job workers
|
|
10
|
+
* - Job processors: Extracted functions for each job type
|
|
11
|
+
* - Job types: All job type definitions
|
|
12
|
+
*/
|
|
13
|
+
export type { JobQueue } from './job-queue-interface';
|
|
14
|
+
export { FsJobQueue } from './fs-job-queue';
|
|
15
|
+
export { JobWorker } from './job-worker';
|
|
16
|
+
export type { JobType, JobStatus, JobMetadata, DetectionJob, GenerationJob, HighlightDetectionJob, AssessmentDetectionJob, CommentDetectionJob, TagDetectionJob, AnyJob, PendingJob, RunningJob, CompleteJob, FailedJob, CancelledJob, JobQueryFilters, DetectionParams, GenerationParams, HighlightDetectionParams, AssessmentDetectionParams, CommentDetectionParams, TagDetectionParams, DetectionProgress, YieldProgress, HighlightDetectionProgress, AssessmentDetectionProgress, CommentDetectionProgress, TagDetectionProgress, DetectionResult, GenerationResult, HighlightDetectionResult, AssessmentDetectionResult, CommentDetectionResult, TagDetectionResult, ContentFetcher, } from './types';
|
|
17
|
+
export { isPendingJob, isRunningJob, isCompleteJob, isFailedJob, isCancelledJob, } from './types';
|
|
18
|
+
export { processHighlightJob, processCommentJob, processAssessmentJob, processReferenceJob, processTagJob, processGenerationJob, type OnProgress, type ProcessorResult, } from './processors';
|
|
19
|
+
export { AnnotationDetection } from './workers/annotation-detection';
|
|
20
|
+
export { generateResourceFromTopic } from './workers/generation/resource-generation';
|
|
21
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,YAAY,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAG5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,YAAY,EACV,OAAO,EACP,SAAS,EACT,WAAW,EACX,YAAY,EACZ,aAAa,EACb,qBAAqB,EACrB,sBAAsB,EACtB,mBAAmB,EACnB,eAAe,EACf,MAAM,EACN,UAAU,EACV,UAAU,EACV,WAAW,EACX,SAAS,EACT,YAAY,EACZ,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,yBAAyB,EACzB,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,aAAa,EACb,0BAA0B,EAC1B,2BAA2B,EAC3B,wBAAwB,EACxB,oBAAoB,EACpB,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,yBAAyB,EACzB,sBAAsB,EACtB,kBAAkB,EAClB,cAAc,GACf,MAAM,SAAS,CAAC;AAEjB,OAAO,EACL,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,WAAW,EACX,cAAc,GACf,MAAM,SAAS,CAAC;AAGjB,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,oBAAoB,EACpB,KAAK,UAAU,EACf,KAAK,eAAe,GACrB,MAAM,cAAc,CAAC;AAGtB,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAGrE,OAAO,EAAE,yBAAyB,EAAE,MAAM,0CAA0C,CAAC"}
|