@semiont/make-meaning 0.2.30-build.60 → 0.2.30-build.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -583
- package/dist/index.d.ts +175 -7
- package/dist/index.js +1410 -54
- package/dist/index.js.map +1 -1
- package/package.json +3 -2
package/dist/index.js
CHANGED
|
@@ -6,11 +6,11 @@ var ResourceContext = class {
|
|
|
6
6
|
/**
|
|
7
7
|
* Get resource metadata from view storage
|
|
8
8
|
*/
|
|
9
|
-
static async getResourceMetadata(
|
|
9
|
+
static async getResourceMetadata(resourceId2, config) {
|
|
10
10
|
const basePath = config.services.filesystem.path;
|
|
11
11
|
const projectRoot = config._metadata?.projectRoot;
|
|
12
12
|
const viewStorage = new FilesystemViewStorage(basePath, projectRoot);
|
|
13
|
-
const view = await viewStorage.get(
|
|
13
|
+
const view = await viewStorage.get(resourceId2);
|
|
14
14
|
if (!view) {
|
|
15
15
|
return null;
|
|
16
16
|
}
|
|
@@ -97,7 +97,7 @@ var AnnotationContext = class {
|
|
|
97
97
|
* @returns Rich context for LLM processing
|
|
98
98
|
* @throws Error if annotation or resource not found
|
|
99
99
|
*/
|
|
100
|
-
static async buildLLMContext(
|
|
100
|
+
static async buildLLMContext(annotationUri2, resourceId2, config, options = {}) {
|
|
101
101
|
const {
|
|
102
102
|
includeSourceContext = true,
|
|
103
103
|
includeTargetContext = true,
|
|
@@ -106,16 +106,16 @@ var AnnotationContext = class {
|
|
|
106
106
|
if (contextWindow < 100 || contextWindow > 5e3) {
|
|
107
107
|
throw new Error("contextWindow must be between 100 and 5000");
|
|
108
108
|
}
|
|
109
|
-
console.log(`[AnnotationContext] buildLLMContext called with annotationUri=${
|
|
109
|
+
console.log(`[AnnotationContext] buildLLMContext called with annotationUri=${annotationUri2}, resourceId=${resourceId2}`);
|
|
110
110
|
const basePath = config.services.filesystem.path;
|
|
111
111
|
console.log(`[AnnotationContext] basePath=${basePath}`);
|
|
112
112
|
const projectRoot = config._metadata?.projectRoot;
|
|
113
113
|
const viewStorage = new FilesystemViewStorage2(basePath, projectRoot);
|
|
114
114
|
const repStore = new FilesystemRepresentationStore2({ basePath }, projectRoot);
|
|
115
|
-
console.log(`[AnnotationContext] Getting view for resourceId=${
|
|
115
|
+
console.log(`[AnnotationContext] Getting view for resourceId=${resourceId2}`);
|
|
116
116
|
let sourceView;
|
|
117
117
|
try {
|
|
118
|
-
sourceView = await viewStorage.get(
|
|
118
|
+
sourceView = await viewStorage.get(resourceId2);
|
|
119
119
|
console.log(`[AnnotationContext] Got view:`, !!sourceView);
|
|
120
120
|
if (!sourceView) {
|
|
121
121
|
throw new Error("Source resource not found");
|
|
@@ -124,19 +124,19 @@ var AnnotationContext = class {
|
|
|
124
124
|
console.error(`[AnnotationContext] Error getting view:`, error);
|
|
125
125
|
throw error;
|
|
126
126
|
}
|
|
127
|
-
console.log(`[AnnotationContext] Looking for annotation ${
|
|
127
|
+
console.log(`[AnnotationContext] Looking for annotation ${annotationUri2} in resource ${resourceId2}`);
|
|
128
128
|
console.log(`[AnnotationContext] View has ${sourceView.annotations.annotations.length} annotations`);
|
|
129
129
|
console.log(`[AnnotationContext] First 5 annotation IDs:`, sourceView.annotations.annotations.slice(0, 5).map((a) => a.id));
|
|
130
|
-
const annotation = sourceView.annotations.annotations.find((a) => a.id ===
|
|
130
|
+
const annotation = sourceView.annotations.annotations.find((a) => a.id === annotationUri2);
|
|
131
131
|
console.log(`[AnnotationContext] Found annotation:`, !!annotation);
|
|
132
132
|
if (!annotation) {
|
|
133
133
|
throw new Error("Annotation not found in view");
|
|
134
134
|
}
|
|
135
135
|
const targetSource = getTargetSource(annotation.target);
|
|
136
136
|
const targetResourceId = targetSource.split("/").pop();
|
|
137
|
-
console.log(`[AnnotationContext] Target source: ${targetSource}, Expected resource ID: ${
|
|
138
|
-
if (targetResourceId !==
|
|
139
|
-
throw new Error(`Annotation target resource ID (${targetResourceId}) does not match expected resource ID (${
|
|
137
|
+
console.log(`[AnnotationContext] Target source: ${targetSource}, Expected resource ID: ${resourceId2}, Extracted ID: ${targetResourceId}`);
|
|
138
|
+
if (targetResourceId !== resourceId2) {
|
|
139
|
+
throw new Error(`Annotation target resource ID (${targetResourceId}) does not match expected resource ID (${resourceId2})`);
|
|
140
140
|
}
|
|
141
141
|
const sourceDoc = sourceView.resource;
|
|
142
142
|
const bodySource = getBodySource(annotation.body);
|
|
@@ -233,16 +233,16 @@ var AnnotationContext = class {
|
|
|
233
233
|
* Get resource annotations from view storage (fast path)
|
|
234
234
|
* Throws if view missing
|
|
235
235
|
*/
|
|
236
|
-
static async getResourceAnnotations(
|
|
236
|
+
static async getResourceAnnotations(resourceId2, config) {
|
|
237
237
|
if (!config.services?.filesystem?.path) {
|
|
238
238
|
throw new Error("Filesystem path not found in configuration");
|
|
239
239
|
}
|
|
240
240
|
const basePath = config.services.filesystem.path;
|
|
241
241
|
const projectRoot = config._metadata?.projectRoot;
|
|
242
242
|
const viewStorage = new FilesystemViewStorage2(basePath, projectRoot);
|
|
243
|
-
const view = await viewStorage.get(
|
|
243
|
+
const view = await viewStorage.get(resourceId2);
|
|
244
244
|
if (!view) {
|
|
245
|
-
throw new Error(`Resource ${
|
|
245
|
+
throw new Error(`Resource ${resourceId2} not found in view storage`);
|
|
246
246
|
}
|
|
247
247
|
return view.annotations;
|
|
248
248
|
}
|
|
@@ -250,8 +250,8 @@ var AnnotationContext = class {
|
|
|
250
250
|
* Get all annotations
|
|
251
251
|
* @returns Array of all annotation objects
|
|
252
252
|
*/
|
|
253
|
-
static async getAllAnnotations(
|
|
254
|
-
const annotations = await this.getResourceAnnotations(
|
|
253
|
+
static async getAllAnnotations(resourceId2, config) {
|
|
254
|
+
const annotations = await this.getResourceAnnotations(resourceId2, config);
|
|
255
255
|
return await this.enrichResolvedReferences(annotations.annotations, config);
|
|
256
256
|
}
|
|
257
257
|
/**
|
|
@@ -328,8 +328,8 @@ var AnnotationContext = class {
|
|
|
328
328
|
* Get resource stats (version info)
|
|
329
329
|
* @returns Version and timestamp info for the annotations
|
|
330
330
|
*/
|
|
331
|
-
static async getResourceStats(
|
|
332
|
-
const annotations = await this.getResourceAnnotations(
|
|
331
|
+
static async getResourceStats(resourceId2, config) {
|
|
332
|
+
const annotations = await this.getResourceAnnotations(resourceId2, config);
|
|
333
333
|
return {
|
|
334
334
|
resourceId: annotations.resourceId,
|
|
335
335
|
version: annotations.version,
|
|
@@ -339,24 +339,24 @@ var AnnotationContext = class {
|
|
|
339
339
|
/**
|
|
340
340
|
* Check if resource exists in view storage
|
|
341
341
|
*/
|
|
342
|
-
static async resourceExists(
|
|
342
|
+
static async resourceExists(resourceId2, config) {
|
|
343
343
|
if (!config.services?.filesystem?.path) {
|
|
344
344
|
throw new Error("Filesystem path not found in configuration");
|
|
345
345
|
}
|
|
346
346
|
const basePath = config.services.filesystem.path;
|
|
347
347
|
const projectRoot = config._metadata?.projectRoot;
|
|
348
348
|
const viewStorage = new FilesystemViewStorage2(basePath, projectRoot);
|
|
349
|
-
return await viewStorage.exists(
|
|
349
|
+
return await viewStorage.exists(resourceId2);
|
|
350
350
|
}
|
|
351
351
|
/**
|
|
352
352
|
* Get a single annotation by ID
|
|
353
353
|
* O(1) lookup using resource ID to access view storage
|
|
354
354
|
*/
|
|
355
|
-
static async getAnnotation(
|
|
356
|
-
const annotations = await this.getResourceAnnotations(
|
|
355
|
+
static async getAnnotation(annotationId2, resourceId2, config) {
|
|
356
|
+
const annotations = await this.getResourceAnnotations(resourceId2, config);
|
|
357
357
|
return annotations.annotations.find((a) => {
|
|
358
358
|
const shortId = a.id.split("/").pop();
|
|
359
|
-
return shortId ===
|
|
359
|
+
return shortId === annotationId2;
|
|
360
360
|
}) || null;
|
|
361
361
|
}
|
|
362
362
|
/**
|
|
@@ -373,11 +373,11 @@ var AnnotationContext = class {
|
|
|
373
373
|
/**
|
|
374
374
|
* Get annotation context (selected text with surrounding context)
|
|
375
375
|
*/
|
|
376
|
-
static async getAnnotationContext(
|
|
376
|
+
static async getAnnotationContext(annotationId2, resourceId2, contextBefore, contextAfter, config) {
|
|
377
377
|
const basePath = config.services.filesystem.path;
|
|
378
378
|
const projectRoot = config._metadata?.projectRoot;
|
|
379
379
|
const repStore = new FilesystemRepresentationStore2({ basePath }, projectRoot);
|
|
380
|
-
const annotation = await this.getAnnotation(
|
|
380
|
+
const annotation = await this.getAnnotation(annotationId2, resourceId2, config);
|
|
381
381
|
if (!annotation) {
|
|
382
382
|
throw new Error("Annotation not found");
|
|
383
383
|
}
|
|
@@ -409,11 +409,11 @@ var AnnotationContext = class {
|
|
|
409
409
|
/**
|
|
410
410
|
* Generate AI summary of annotation in context
|
|
411
411
|
*/
|
|
412
|
-
static async generateAnnotationSummary(
|
|
412
|
+
static async generateAnnotationSummary(annotationId2, resourceId2, config) {
|
|
413
413
|
const basePath = config.services.filesystem.path;
|
|
414
414
|
const projectRoot = config._metadata?.projectRoot;
|
|
415
415
|
const repStore = new FilesystemRepresentationStore2({ basePath }, projectRoot);
|
|
416
|
-
const annotation = await this.getAnnotation(
|
|
416
|
+
const annotation = await this.getAnnotation(annotationId2, resourceId2, config);
|
|
417
417
|
if (!annotation) {
|
|
418
418
|
throw new Error("Annotation not found");
|
|
419
419
|
}
|
|
@@ -499,10 +499,10 @@ var GraphContext = class {
|
|
|
499
499
|
* Get all resources referencing this resource (backlinks)
|
|
500
500
|
* Requires graph traversal - must use graph database
|
|
501
501
|
*/
|
|
502
|
-
static async getBacklinks(
|
|
502
|
+
static async getBacklinks(resourceId2, config) {
|
|
503
503
|
const graphDb = await getGraphDatabase(config);
|
|
504
|
-
const
|
|
505
|
-
return await graphDb.getResourceReferencedBy(
|
|
504
|
+
const resourceUri2 = resourceIdToURI(resourceId2, config.services.backend.publicURL);
|
|
505
|
+
return await graphDb.getResourceReferencedBy(resourceUri2);
|
|
506
506
|
}
|
|
507
507
|
/**
|
|
508
508
|
* Find shortest path between two resources
|
|
@@ -516,9 +516,9 @@ var GraphContext = class {
|
|
|
516
516
|
* Get resource connections (graph edges)
|
|
517
517
|
* Requires graph traversal - must use graph database
|
|
518
518
|
*/
|
|
519
|
-
static async getResourceConnections(
|
|
519
|
+
static async getResourceConnections(resourceId2, config) {
|
|
520
520
|
const graphDb = await getGraphDatabase(config);
|
|
521
|
-
return await graphDb.getResourceConnections(
|
|
521
|
+
return await graphDb.getResourceConnections(resourceId2);
|
|
522
522
|
}
|
|
523
523
|
/**
|
|
524
524
|
* Search resources by name (cross-resource query)
|
|
@@ -550,14 +550,14 @@ var AnnotationDetection = class {
|
|
|
550
550
|
* @param density - Optional target number of comments per 2000 words
|
|
551
551
|
* @returns Array of validated comment matches
|
|
552
552
|
*/
|
|
553
|
-
static async detectComments(
|
|
554
|
-
const resource = await ResourceContext.getResourceMetadata(
|
|
553
|
+
static async detectComments(resourceId2, config, instructions, tone, density) {
|
|
554
|
+
const resource = await ResourceContext.getResourceMetadata(resourceId2, config);
|
|
555
555
|
if (!resource) {
|
|
556
|
-
throw new Error(`Resource ${
|
|
556
|
+
throw new Error(`Resource ${resourceId2} not found`);
|
|
557
557
|
}
|
|
558
|
-
const content = await this.loadResourceContent(
|
|
558
|
+
const content = await this.loadResourceContent(resourceId2, config);
|
|
559
559
|
if (!content) {
|
|
560
|
-
throw new Error(`Could not load content for resource ${
|
|
560
|
+
throw new Error(`Could not load content for resource ${resourceId2}`);
|
|
561
561
|
}
|
|
562
562
|
const prompt = MotivationPrompts.buildCommentPrompt(content, instructions, tone, density);
|
|
563
563
|
const response = await generateText2(
|
|
@@ -579,14 +579,14 @@ var AnnotationDetection = class {
|
|
|
579
579
|
* @param density - Optional target number of highlights per 2000 words
|
|
580
580
|
* @returns Array of validated highlight matches
|
|
581
581
|
*/
|
|
582
|
-
static async detectHighlights(
|
|
583
|
-
const resource = await ResourceContext.getResourceMetadata(
|
|
582
|
+
static async detectHighlights(resourceId2, config, instructions, density) {
|
|
583
|
+
const resource = await ResourceContext.getResourceMetadata(resourceId2, config);
|
|
584
584
|
if (!resource) {
|
|
585
|
-
throw new Error(`Resource ${
|
|
585
|
+
throw new Error(`Resource ${resourceId2} not found`);
|
|
586
586
|
}
|
|
587
|
-
const content = await this.loadResourceContent(
|
|
587
|
+
const content = await this.loadResourceContent(resourceId2, config);
|
|
588
588
|
if (!content) {
|
|
589
|
-
throw new Error(`Could not load content for resource ${
|
|
589
|
+
throw new Error(`Could not load content for resource ${resourceId2}`);
|
|
590
590
|
}
|
|
591
591
|
const prompt = MotivationPrompts.buildHighlightPrompt(content, instructions, density);
|
|
592
592
|
const response = await generateText2(
|
|
@@ -609,14 +609,14 @@ var AnnotationDetection = class {
|
|
|
609
609
|
* @param density - Optional target number of assessments per 2000 words
|
|
610
610
|
* @returns Array of validated assessment matches
|
|
611
611
|
*/
|
|
612
|
-
static async detectAssessments(
|
|
613
|
-
const resource = await ResourceContext.getResourceMetadata(
|
|
612
|
+
static async detectAssessments(resourceId2, config, instructions, tone, density) {
|
|
613
|
+
const resource = await ResourceContext.getResourceMetadata(resourceId2, config);
|
|
614
614
|
if (!resource) {
|
|
615
|
-
throw new Error(`Resource ${
|
|
615
|
+
throw new Error(`Resource ${resourceId2} not found`);
|
|
616
616
|
}
|
|
617
|
-
const content = await this.loadResourceContent(
|
|
617
|
+
const content = await this.loadResourceContent(resourceId2, config);
|
|
618
618
|
if (!content) {
|
|
619
|
-
throw new Error(`Could not load content for resource ${
|
|
619
|
+
throw new Error(`Could not load content for resource ${resourceId2}`);
|
|
620
620
|
}
|
|
621
621
|
const prompt = MotivationPrompts.buildAssessmentPrompt(content, instructions, tone, density);
|
|
622
622
|
const response = await generateText2(
|
|
@@ -638,7 +638,7 @@ var AnnotationDetection = class {
|
|
|
638
638
|
* @param category - The specific category to detect
|
|
639
639
|
* @returns Array of validated tag matches
|
|
640
640
|
*/
|
|
641
|
-
static async detectTags(
|
|
641
|
+
static async detectTags(resourceId2, config, schemaId, category) {
|
|
642
642
|
const schema = getTagSchema(schemaId);
|
|
643
643
|
if (!schema) {
|
|
644
644
|
throw new Error(`Invalid tag schema: ${schemaId}`);
|
|
@@ -647,13 +647,13 @@ var AnnotationDetection = class {
|
|
|
647
647
|
if (!categoryInfo) {
|
|
648
648
|
throw new Error(`Invalid category "${category}" for schema ${schemaId}`);
|
|
649
649
|
}
|
|
650
|
-
const resource = await ResourceContext.getResourceMetadata(
|
|
650
|
+
const resource = await ResourceContext.getResourceMetadata(resourceId2, config);
|
|
651
651
|
if (!resource) {
|
|
652
|
-
throw new Error(`Resource ${
|
|
652
|
+
throw new Error(`Resource ${resourceId2} not found`);
|
|
653
653
|
}
|
|
654
|
-
const content = await this.loadResourceContent(
|
|
654
|
+
const content = await this.loadResourceContent(resourceId2, config);
|
|
655
655
|
if (!content) {
|
|
656
|
-
throw new Error(`Could not load content for resource ${
|
|
656
|
+
throw new Error(`Could not load content for resource ${resourceId2}`);
|
|
657
657
|
}
|
|
658
658
|
const prompt = MotivationPrompts.buildTagPrompt(
|
|
659
659
|
content,
|
|
@@ -683,8 +683,8 @@ var AnnotationDetection = class {
|
|
|
683
683
|
* @param config - Environment configuration
|
|
684
684
|
* @returns Resource content as string, or null if not available
|
|
685
685
|
*/
|
|
686
|
-
static async loadResourceContent(
|
|
687
|
-
const resource = await ResourceContext.getResourceMetadata(
|
|
686
|
+
static async loadResourceContent(resourceId2, config) {
|
|
687
|
+
const resource = await ResourceContext.getResourceMetadata(resourceId2, config);
|
|
688
688
|
if (!resource) return null;
|
|
689
689
|
const primaryRep = getPrimaryRepresentation3(resource);
|
|
690
690
|
if (!primaryRep) return null;
|
|
@@ -701,15 +701,1371 @@ var AnnotationDetection = class {
|
|
|
701
701
|
}
|
|
702
702
|
};
|
|
703
703
|
|
|
704
|
+
// src/jobs/workers/comment-detection-worker.ts
|
|
705
|
+
import { JobWorker } from "@semiont/jobs";
|
|
706
|
+
import { generateAnnotationId } from "@semiont/event-sourcing";
|
|
707
|
+
import { resourceIdToURI as resourceIdToURI2 } from "@semiont/core";
|
|
708
|
+
import { userId } from "@semiont/core";
|
|
709
|
+
var CommentDetectionWorker = class extends JobWorker {
|
|
710
|
+
constructor(jobQueue, config, eventStore) {
|
|
711
|
+
super(jobQueue);
|
|
712
|
+
this.config = config;
|
|
713
|
+
this.eventStore = eventStore;
|
|
714
|
+
}
|
|
715
|
+
isFirstProgress = true;
|
|
716
|
+
getWorkerName() {
|
|
717
|
+
return "CommentDetectionWorker";
|
|
718
|
+
}
|
|
719
|
+
canProcessJob(job) {
|
|
720
|
+
return job.metadata.type === "comment-detection";
|
|
721
|
+
}
|
|
722
|
+
async executeJob(job) {
|
|
723
|
+
if (job.metadata.type !== "comment-detection") {
|
|
724
|
+
throw new Error(`Invalid job type: ${job.metadata.type}`);
|
|
725
|
+
}
|
|
726
|
+
if (job.status !== "running") {
|
|
727
|
+
throw new Error(`Job must be in running state to execute, got: ${job.status}`);
|
|
728
|
+
}
|
|
729
|
+
this.isFirstProgress = true;
|
|
730
|
+
await this.processCommentDetectionJob(job);
|
|
731
|
+
}
|
|
732
|
+
/**
|
|
733
|
+
* Override updateJobProgress to emit events to Event Store
|
|
734
|
+
*/
|
|
735
|
+
async updateJobProgress(job) {
|
|
736
|
+
await super.updateJobProgress(job);
|
|
737
|
+
if (job.metadata.type !== "comment-detection") return;
|
|
738
|
+
if (job.status !== "running") {
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
const cdJob = job;
|
|
742
|
+
const baseEvent = {
|
|
743
|
+
resourceId: cdJob.params.resourceId,
|
|
744
|
+
userId: cdJob.metadata.userId,
|
|
745
|
+
version: 1
|
|
746
|
+
};
|
|
747
|
+
const isComplete = cdJob.progress.percentage === 100;
|
|
748
|
+
if (this.isFirstProgress) {
|
|
749
|
+
this.isFirstProgress = false;
|
|
750
|
+
await this.eventStore.appendEvent({
|
|
751
|
+
type: "job.started",
|
|
752
|
+
...baseEvent,
|
|
753
|
+
payload: {
|
|
754
|
+
jobId: cdJob.metadata.id,
|
|
755
|
+
jobType: cdJob.metadata.type
|
|
756
|
+
}
|
|
757
|
+
});
|
|
758
|
+
} else if (isComplete) {
|
|
759
|
+
await this.eventStore.appendEvent({
|
|
760
|
+
type: "job.completed",
|
|
761
|
+
...baseEvent,
|
|
762
|
+
payload: {
|
|
763
|
+
jobId: cdJob.metadata.id,
|
|
764
|
+
jobType: cdJob.metadata.type
|
|
765
|
+
// Note: result would come from job.result, but that's handled by base class
|
|
766
|
+
}
|
|
767
|
+
});
|
|
768
|
+
} else {
|
|
769
|
+
await this.eventStore.appendEvent({
|
|
770
|
+
type: "job.progress",
|
|
771
|
+
...baseEvent,
|
|
772
|
+
payload: {
|
|
773
|
+
jobId: cdJob.metadata.id,
|
|
774
|
+
jobType: cdJob.metadata.type,
|
|
775
|
+
progress: cdJob.progress
|
|
776
|
+
}
|
|
777
|
+
});
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
async handleJobFailure(job, error) {
|
|
781
|
+
await super.handleJobFailure(job, error);
|
|
782
|
+
if (job.status === "failed" && job.metadata.type === "comment-detection") {
|
|
783
|
+
const cdJob = job;
|
|
784
|
+
await this.eventStore.appendEvent({
|
|
785
|
+
type: "job.failed",
|
|
786
|
+
resourceId: cdJob.params.resourceId,
|
|
787
|
+
userId: cdJob.metadata.userId,
|
|
788
|
+
version: 1,
|
|
789
|
+
payload: {
|
|
790
|
+
jobId: cdJob.metadata.id,
|
|
791
|
+
jobType: cdJob.metadata.type,
|
|
792
|
+
error: "Comment detection failed. Please try again later."
|
|
793
|
+
}
|
|
794
|
+
});
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
async processCommentDetectionJob(job) {
|
|
798
|
+
console.log(`[CommentDetectionWorker] Processing comment detection for resource ${job.params.resourceId} (job: ${job.metadata.id})`);
|
|
799
|
+
const resource = await ResourceContext.getResourceMetadata(job.params.resourceId, this.config);
|
|
800
|
+
if (!resource) {
|
|
801
|
+
throw new Error(`Resource ${job.params.resourceId} not found`);
|
|
802
|
+
}
|
|
803
|
+
let updatedJob = {
|
|
804
|
+
...job,
|
|
805
|
+
progress: {
|
|
806
|
+
stage: "analyzing",
|
|
807
|
+
percentage: 10,
|
|
808
|
+
message: "Loading resource..."
|
|
809
|
+
}
|
|
810
|
+
};
|
|
811
|
+
await this.updateJobProgress(updatedJob);
|
|
812
|
+
updatedJob = {
|
|
813
|
+
...updatedJob,
|
|
814
|
+
progress: {
|
|
815
|
+
stage: "analyzing",
|
|
816
|
+
percentage: 30,
|
|
817
|
+
message: "Analyzing text and generating comments..."
|
|
818
|
+
}
|
|
819
|
+
};
|
|
820
|
+
await this.updateJobProgress(updatedJob);
|
|
821
|
+
const comments = await AnnotationDetection.detectComments(
|
|
822
|
+
job.params.resourceId,
|
|
823
|
+
this.config,
|
|
824
|
+
job.params.instructions,
|
|
825
|
+
job.params.tone,
|
|
826
|
+
job.params.density
|
|
827
|
+
);
|
|
828
|
+
console.log(`[CommentDetectionWorker] Found ${comments.length} comments to create`);
|
|
829
|
+
updatedJob = {
|
|
830
|
+
...updatedJob,
|
|
831
|
+
progress: {
|
|
832
|
+
stage: "creating",
|
|
833
|
+
percentage: 60,
|
|
834
|
+
message: `Creating ${comments.length} annotations...`
|
|
835
|
+
}
|
|
836
|
+
};
|
|
837
|
+
await this.updateJobProgress(updatedJob);
|
|
838
|
+
let created = 0;
|
|
839
|
+
for (const comment of comments) {
|
|
840
|
+
try {
|
|
841
|
+
await this.createCommentAnnotation(job.params.resourceId, job.metadata.userId, comment);
|
|
842
|
+
created++;
|
|
843
|
+
} catch (error) {
|
|
844
|
+
console.error(`[CommentDetectionWorker] Failed to create comment:`, error);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
updatedJob = {
|
|
848
|
+
...updatedJob,
|
|
849
|
+
progress: {
|
|
850
|
+
stage: "creating",
|
|
851
|
+
percentage: 100,
|
|
852
|
+
message: `Complete! Created ${created} comments`
|
|
853
|
+
}
|
|
854
|
+
};
|
|
855
|
+
await this.updateJobProgress(updatedJob);
|
|
856
|
+
console.log(`[CommentDetectionWorker] \u2705 Created ${created}/${comments.length} comments`);
|
|
857
|
+
}
|
|
858
|
+
async createCommentAnnotation(resourceId2, userId_, comment) {
|
|
859
|
+
const backendUrl = this.config.services.backend?.publicURL;
|
|
860
|
+
if (!backendUrl) {
|
|
861
|
+
throw new Error("Backend publicURL not configured");
|
|
862
|
+
}
|
|
863
|
+
const resourceUri2 = resourceIdToURI2(resourceId2, backendUrl);
|
|
864
|
+
const annotationId2 = generateAnnotationId(backendUrl);
|
|
865
|
+
const annotation = {
|
|
866
|
+
"@context": "http://www.w3.org/ns/anno.jsonld",
|
|
867
|
+
type: "Annotation",
|
|
868
|
+
id: annotationId2,
|
|
869
|
+
motivation: "commenting",
|
|
870
|
+
target: {
|
|
871
|
+
type: "SpecificResource",
|
|
872
|
+
source: resourceUri2,
|
|
873
|
+
selector: [
|
|
874
|
+
{
|
|
875
|
+
type: "TextPositionSelector",
|
|
876
|
+
start: comment.start,
|
|
877
|
+
end: comment.end
|
|
878
|
+
},
|
|
879
|
+
{
|
|
880
|
+
type: "TextQuoteSelector",
|
|
881
|
+
exact: comment.exact,
|
|
882
|
+
prefix: comment.prefix || "",
|
|
883
|
+
suffix: comment.suffix || ""
|
|
884
|
+
}
|
|
885
|
+
]
|
|
886
|
+
},
|
|
887
|
+
body: [
|
|
888
|
+
{
|
|
889
|
+
type: "TextualBody",
|
|
890
|
+
value: comment.comment,
|
|
891
|
+
purpose: "commenting",
|
|
892
|
+
format: "text/plain",
|
|
893
|
+
language: "en"
|
|
894
|
+
}
|
|
895
|
+
]
|
|
896
|
+
};
|
|
897
|
+
await this.eventStore.appendEvent({
|
|
898
|
+
type: "annotation.added",
|
|
899
|
+
resourceId: resourceId2,
|
|
900
|
+
userId: userId(userId_),
|
|
901
|
+
version: 1,
|
|
902
|
+
payload: {
|
|
903
|
+
annotation
|
|
904
|
+
}
|
|
905
|
+
});
|
|
906
|
+
console.log(`[CommentDetectionWorker] Created comment annotation ${annotationId2} for "${comment.exact.substring(0, 50)}..."`);
|
|
907
|
+
}
|
|
908
|
+
};
|
|
909
|
+
|
|
910
|
+
// src/jobs/workers/highlight-detection-worker.ts
|
|
911
|
+
import { JobWorker as JobWorker2 } from "@semiont/jobs";
|
|
912
|
+
import { generateAnnotationId as generateAnnotationId2 } from "@semiont/event-sourcing";
|
|
913
|
+
import { resourceIdToURI as resourceIdToURI3 } from "@semiont/core";
|
|
914
|
+
import { userId as userId2 } from "@semiont/core";
|
|
915
|
+
var HighlightDetectionWorker = class extends JobWorker2 {
|
|
916
|
+
constructor(jobQueue, config, eventStore) {
|
|
917
|
+
super(jobQueue);
|
|
918
|
+
this.config = config;
|
|
919
|
+
this.eventStore = eventStore;
|
|
920
|
+
}
|
|
921
|
+
isFirstProgress = true;
|
|
922
|
+
getWorkerName() {
|
|
923
|
+
return "HighlightDetectionWorker";
|
|
924
|
+
}
|
|
925
|
+
canProcessJob(job) {
|
|
926
|
+
return job.metadata.type === "highlight-detection";
|
|
927
|
+
}
|
|
928
|
+
async executeJob(job) {
|
|
929
|
+
if (job.metadata.type !== "highlight-detection") {
|
|
930
|
+
throw new Error(`Invalid job type: ${job.metadata.type}`);
|
|
931
|
+
}
|
|
932
|
+
if (job.status !== "running") {
|
|
933
|
+
throw new Error(`Job must be in running state to execute, got: ${job.status}`);
|
|
934
|
+
}
|
|
935
|
+
this.isFirstProgress = true;
|
|
936
|
+
await this.processHighlightDetectionJob(job);
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Override updateJobProgress to emit events to Event Store
|
|
940
|
+
*/
|
|
941
|
+
async updateJobProgress(job) {
|
|
942
|
+
await super.updateJobProgress(job);
|
|
943
|
+
if (job.metadata.type !== "highlight-detection") return;
|
|
944
|
+
if (job.status !== "running") {
|
|
945
|
+
return;
|
|
946
|
+
}
|
|
947
|
+
const hlJob = job;
|
|
948
|
+
const baseEvent = {
|
|
949
|
+
resourceId: hlJob.params.resourceId,
|
|
950
|
+
userId: hlJob.metadata.userId,
|
|
951
|
+
version: 1
|
|
952
|
+
};
|
|
953
|
+
const isComplete = hlJob.progress.percentage === 100;
|
|
954
|
+
if (this.isFirstProgress) {
|
|
955
|
+
this.isFirstProgress = false;
|
|
956
|
+
await this.eventStore.appendEvent({
|
|
957
|
+
type: "job.started",
|
|
958
|
+
...baseEvent,
|
|
959
|
+
payload: {
|
|
960
|
+
jobId: hlJob.metadata.id,
|
|
961
|
+
jobType: hlJob.metadata.type
|
|
962
|
+
}
|
|
963
|
+
});
|
|
964
|
+
} else if (isComplete) {
|
|
965
|
+
await this.eventStore.appendEvent({
|
|
966
|
+
type: "job.completed",
|
|
967
|
+
...baseEvent,
|
|
968
|
+
payload: {
|
|
969
|
+
jobId: hlJob.metadata.id,
|
|
970
|
+
jobType: hlJob.metadata.type
|
|
971
|
+
// Note: result would come from job.result, but that's handled by base class
|
|
972
|
+
}
|
|
973
|
+
});
|
|
974
|
+
} else {
|
|
975
|
+
await this.eventStore.appendEvent({
|
|
976
|
+
type: "job.progress",
|
|
977
|
+
...baseEvent,
|
|
978
|
+
payload: {
|
|
979
|
+
jobId: hlJob.metadata.id,
|
|
980
|
+
jobType: hlJob.metadata.type,
|
|
981
|
+
progress: hlJob.progress
|
|
982
|
+
}
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
async handleJobFailure(job, error) {
|
|
987
|
+
await super.handleJobFailure(job, error);
|
|
988
|
+
if (job.status === "failed" && job.metadata.type === "highlight-detection") {
|
|
989
|
+
const hlJob = job;
|
|
990
|
+
await this.eventStore.appendEvent({
|
|
991
|
+
type: "job.failed",
|
|
992
|
+
resourceId: hlJob.params.resourceId,
|
|
993
|
+
userId: hlJob.metadata.userId,
|
|
994
|
+
version: 1,
|
|
995
|
+
payload: {
|
|
996
|
+
jobId: hlJob.metadata.id,
|
|
997
|
+
jobType: hlJob.metadata.type,
|
|
998
|
+
error: "Highlight detection failed. Please try again later."
|
|
999
|
+
}
|
|
1000
|
+
});
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
async processHighlightDetectionJob(job) {
|
|
1004
|
+
console.log(`[HighlightDetectionWorker] Processing highlight detection for resource ${job.params.resourceId} (job: ${job.metadata.id})`);
|
|
1005
|
+
const resource = await ResourceContext.getResourceMetadata(job.params.resourceId, this.config);
|
|
1006
|
+
if (!resource) {
|
|
1007
|
+
throw new Error(`Resource ${job.params.resourceId} not found`);
|
|
1008
|
+
}
|
|
1009
|
+
let updatedJob = {
|
|
1010
|
+
...job,
|
|
1011
|
+
progress: {
|
|
1012
|
+
stage: "analyzing",
|
|
1013
|
+
percentage: 10,
|
|
1014
|
+
message: "Loading resource..."
|
|
1015
|
+
}
|
|
1016
|
+
};
|
|
1017
|
+
await this.updateJobProgress(updatedJob);
|
|
1018
|
+
updatedJob = {
|
|
1019
|
+
...updatedJob,
|
|
1020
|
+
progress: {
|
|
1021
|
+
stage: "analyzing",
|
|
1022
|
+
percentage: 30,
|
|
1023
|
+
message: "Analyzing text..."
|
|
1024
|
+
}
|
|
1025
|
+
};
|
|
1026
|
+
await this.updateJobProgress(updatedJob);
|
|
1027
|
+
const highlights = await AnnotationDetection.detectHighlights(
|
|
1028
|
+
job.params.resourceId,
|
|
1029
|
+
this.config,
|
|
1030
|
+
job.params.instructions,
|
|
1031
|
+
job.params.density
|
|
1032
|
+
);
|
|
1033
|
+
console.log(`[HighlightDetectionWorker] Found ${highlights.length} highlights to create`);
|
|
1034
|
+
updatedJob = {
|
|
1035
|
+
...updatedJob,
|
|
1036
|
+
progress: {
|
|
1037
|
+
stage: "creating",
|
|
1038
|
+
percentage: 60,
|
|
1039
|
+
message: `Creating ${highlights.length} annotations...`
|
|
1040
|
+
}
|
|
1041
|
+
};
|
|
1042
|
+
await this.updateJobProgress(updatedJob);
|
|
1043
|
+
let created = 0;
|
|
1044
|
+
for (const highlight of highlights) {
|
|
1045
|
+
try {
|
|
1046
|
+
await this.createHighlightAnnotation(job.params.resourceId, job.metadata.userId, highlight);
|
|
1047
|
+
created++;
|
|
1048
|
+
} catch (error) {
|
|
1049
|
+
console.error(`[HighlightDetectionWorker] Failed to create highlight:`, error);
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
updatedJob = {
|
|
1053
|
+
...updatedJob,
|
|
1054
|
+
progress: {
|
|
1055
|
+
stage: "creating",
|
|
1056
|
+
percentage: 100,
|
|
1057
|
+
message: `Complete! Created ${created} highlights`
|
|
1058
|
+
}
|
|
1059
|
+
};
|
|
1060
|
+
await this.updateJobProgress(updatedJob);
|
|
1061
|
+
console.log(`[HighlightDetectionWorker] \u2705 Created ${created}/${highlights.length} highlights`);
|
|
1062
|
+
}
|
|
1063
|
+
async createHighlightAnnotation(resourceId2, creatorUserId, highlight) {
|
|
1064
|
+
const backendUrl = this.config.services.backend?.publicURL;
|
|
1065
|
+
if (!backendUrl) throw new Error("Backend publicURL not configured");
|
|
1066
|
+
const annotationId2 = generateAnnotationId2(backendUrl);
|
|
1067
|
+
const resourceUri2 = resourceIdToURI3(resourceId2, backendUrl);
|
|
1068
|
+
const annotation = {
|
|
1069
|
+
"@context": "http://www.w3.org/ns/anno.jsonld",
|
|
1070
|
+
"type": "Annotation",
|
|
1071
|
+
"id": annotationId2,
|
|
1072
|
+
"motivation": "highlighting",
|
|
1073
|
+
"creator": userId2(creatorUserId),
|
|
1074
|
+
"created": (/* @__PURE__ */ new Date()).toISOString(),
|
|
1075
|
+
"target": {
|
|
1076
|
+
type: "SpecificResource",
|
|
1077
|
+
source: resourceUri2,
|
|
1078
|
+
selector: [
|
|
1079
|
+
{
|
|
1080
|
+
type: "TextPositionSelector",
|
|
1081
|
+
start: highlight.start,
|
|
1082
|
+
end: highlight.end
|
|
1083
|
+
},
|
|
1084
|
+
{
|
|
1085
|
+
type: "TextQuoteSelector",
|
|
1086
|
+
exact: highlight.exact,
|
|
1087
|
+
...highlight.prefix && { prefix: highlight.prefix },
|
|
1088
|
+
...highlight.suffix && { suffix: highlight.suffix }
|
|
1089
|
+
}
|
|
1090
|
+
]
|
|
1091
|
+
},
|
|
1092
|
+
"body": []
|
|
1093
|
+
// Empty body for highlights
|
|
1094
|
+
};
|
|
1095
|
+
await this.eventStore.appendEvent({
|
|
1096
|
+
type: "annotation.added",
|
|
1097
|
+
resourceId: resourceId2,
|
|
1098
|
+
userId: userId2(creatorUserId),
|
|
1099
|
+
version: 1,
|
|
1100
|
+
payload: { annotation }
|
|
1101
|
+
});
|
|
1102
|
+
}
|
|
1103
|
+
};
|
|
1104
|
+
|
|
1105
|
+
// src/jobs/workers/assessment-detection-worker.ts
|
|
1106
|
+
import { JobWorker as JobWorker3 } from "@semiont/jobs";
|
|
1107
|
+
import { generateAnnotationId as generateAnnotationId3 } from "@semiont/event-sourcing";
|
|
1108
|
+
import { resourceIdToURI as resourceIdToURI4 } from "@semiont/core";
|
|
1109
|
+
import { userId as userId3 } from "@semiont/core";
|
|
1110
|
+
var AssessmentDetectionWorker = class extends JobWorker3 {
|
|
1111
|
+
constructor(jobQueue, config, eventStore) {
|
|
1112
|
+
super(jobQueue);
|
|
1113
|
+
this.config = config;
|
|
1114
|
+
this.eventStore = eventStore;
|
|
1115
|
+
}
|
|
1116
|
+
isFirstProgress = true;
|
|
1117
|
+
getWorkerName() {
|
|
1118
|
+
return "AssessmentDetectionWorker";
|
|
1119
|
+
}
|
|
1120
|
+
canProcessJob(job) {
|
|
1121
|
+
return job.metadata.type === "assessment-detection";
|
|
1122
|
+
}
|
|
1123
|
+
async executeJob(job) {
|
|
1124
|
+
if (job.metadata.type !== "assessment-detection") {
|
|
1125
|
+
throw new Error(`Invalid job type: ${job.metadata.type}`);
|
|
1126
|
+
}
|
|
1127
|
+
if (job.status !== "running") {
|
|
1128
|
+
throw new Error(`Job must be in running state to execute, got: ${job.status}`);
|
|
1129
|
+
}
|
|
1130
|
+
this.isFirstProgress = true;
|
|
1131
|
+
await this.processAssessmentDetectionJob(job);
|
|
1132
|
+
}
|
|
1133
|
+
/**
|
|
1134
|
+
* Override updateJobProgress to emit events to Event Store
|
|
1135
|
+
*/
|
|
1136
|
+
async updateJobProgress(job) {
|
|
1137
|
+
await super.updateJobProgress(job);
|
|
1138
|
+
if (job.metadata.type !== "assessment-detection") return;
|
|
1139
|
+
if (job.status !== "running") {
|
|
1140
|
+
return;
|
|
1141
|
+
}
|
|
1142
|
+
const assJob = job;
|
|
1143
|
+
const baseEvent = {
|
|
1144
|
+
resourceId: assJob.params.resourceId,
|
|
1145
|
+
userId: assJob.metadata.userId,
|
|
1146
|
+
version: 1
|
|
1147
|
+
};
|
|
1148
|
+
const isComplete = assJob.progress.percentage === 100;
|
|
1149
|
+
if (this.isFirstProgress) {
|
|
1150
|
+
this.isFirstProgress = false;
|
|
1151
|
+
await this.eventStore.appendEvent({
|
|
1152
|
+
type: "job.started",
|
|
1153
|
+
...baseEvent,
|
|
1154
|
+
payload: {
|
|
1155
|
+
jobId: assJob.metadata.id,
|
|
1156
|
+
jobType: assJob.metadata.type
|
|
1157
|
+
}
|
|
1158
|
+
});
|
|
1159
|
+
} else if (isComplete) {
|
|
1160
|
+
await this.eventStore.appendEvent({
|
|
1161
|
+
type: "job.completed",
|
|
1162
|
+
...baseEvent,
|
|
1163
|
+
payload: {
|
|
1164
|
+
jobId: assJob.metadata.id,
|
|
1165
|
+
jobType: assJob.metadata.type
|
|
1166
|
+
// Note: result would come from job.result, but that's handled by base class
|
|
1167
|
+
}
|
|
1168
|
+
});
|
|
1169
|
+
} else {
|
|
1170
|
+
await this.eventStore.appendEvent({
|
|
1171
|
+
type: "job.progress",
|
|
1172
|
+
...baseEvent,
|
|
1173
|
+
payload: {
|
|
1174
|
+
jobId: assJob.metadata.id,
|
|
1175
|
+
jobType: assJob.metadata.type,
|
|
1176
|
+
progress: assJob.progress
|
|
1177
|
+
}
|
|
1178
|
+
});
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
async handleJobFailure(job, error) {
|
|
1182
|
+
await super.handleJobFailure(job, error);
|
|
1183
|
+
if (job.status === "failed" && job.metadata.type === "assessment-detection") {
|
|
1184
|
+
const aJob = job;
|
|
1185
|
+
await this.eventStore.appendEvent({
|
|
1186
|
+
type: "job.failed",
|
|
1187
|
+
resourceId: aJob.params.resourceId,
|
|
1188
|
+
userId: aJob.metadata.userId,
|
|
1189
|
+
version: 1,
|
|
1190
|
+
payload: {
|
|
1191
|
+
jobId: aJob.metadata.id,
|
|
1192
|
+
jobType: aJob.metadata.type,
|
|
1193
|
+
error: "Assessment detection failed. Please try again later."
|
|
1194
|
+
}
|
|
1195
|
+
});
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
async processAssessmentDetectionJob(job) {
|
|
1199
|
+
console.log(`[AssessmentDetectionWorker] Processing assessment detection for resource ${job.params.resourceId} (job: ${job.metadata.id})`);
|
|
1200
|
+
const resource = await ResourceContext.getResourceMetadata(job.params.resourceId, this.config);
|
|
1201
|
+
if (!resource) {
|
|
1202
|
+
throw new Error(`Resource ${job.params.resourceId} not found`);
|
|
1203
|
+
}
|
|
1204
|
+
let updatedJob = {
|
|
1205
|
+
...job,
|
|
1206
|
+
progress: {
|
|
1207
|
+
stage: "analyzing",
|
|
1208
|
+
percentage: 10,
|
|
1209
|
+
message: "Loading resource..."
|
|
1210
|
+
}
|
|
1211
|
+
};
|
|
1212
|
+
await this.updateJobProgress(updatedJob);
|
|
1213
|
+
updatedJob = {
|
|
1214
|
+
...updatedJob,
|
|
1215
|
+
progress: {
|
|
1216
|
+
stage: "analyzing",
|
|
1217
|
+
percentage: 30,
|
|
1218
|
+
message: "Analyzing text..."
|
|
1219
|
+
}
|
|
1220
|
+
};
|
|
1221
|
+
await this.updateJobProgress(updatedJob);
|
|
1222
|
+
const assessments = await AnnotationDetection.detectAssessments(
|
|
1223
|
+
job.params.resourceId,
|
|
1224
|
+
this.config,
|
|
1225
|
+
job.params.instructions,
|
|
1226
|
+
job.params.tone,
|
|
1227
|
+
job.params.density
|
|
1228
|
+
);
|
|
1229
|
+
console.log(`[AssessmentDetectionWorker] Found ${assessments.length} assessments to create`);
|
|
1230
|
+
updatedJob = {
|
|
1231
|
+
...updatedJob,
|
|
1232
|
+
progress: {
|
|
1233
|
+
stage: "creating",
|
|
1234
|
+
percentage: 60,
|
|
1235
|
+
message: `Creating ${assessments.length} annotations...`
|
|
1236
|
+
}
|
|
1237
|
+
};
|
|
1238
|
+
await this.updateJobProgress(updatedJob);
|
|
1239
|
+
let created = 0;
|
|
1240
|
+
for (const assessment of assessments) {
|
|
1241
|
+
try {
|
|
1242
|
+
await this.createAssessmentAnnotation(job.params.resourceId, job.metadata.userId, assessment);
|
|
1243
|
+
created++;
|
|
1244
|
+
} catch (error) {
|
|
1245
|
+
console.error(`[AssessmentDetectionWorker] Failed to create assessment:`, error);
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
1248
|
+
updatedJob = {
|
|
1249
|
+
...updatedJob,
|
|
1250
|
+
progress: {
|
|
1251
|
+
stage: "creating",
|
|
1252
|
+
percentage: 100,
|
|
1253
|
+
message: `Complete! Created ${created} assessments`
|
|
1254
|
+
}
|
|
1255
|
+
};
|
|
1256
|
+
await this.updateJobProgress(updatedJob);
|
|
1257
|
+
console.log(`[AssessmentDetectionWorker] \u2705 Created ${created}/${assessments.length} assessments`);
|
|
1258
|
+
}
|
|
1259
|
+
async createAssessmentAnnotation(resourceId2, creatorUserId, assessment) {
|
|
1260
|
+
const backendUrl = this.config.services.backend?.publicURL;
|
|
1261
|
+
if (!backendUrl) throw new Error("Backend publicURL not configured");
|
|
1262
|
+
const annotationId2 = generateAnnotationId3(backendUrl);
|
|
1263
|
+
const resourceUri2 = resourceIdToURI4(resourceId2, backendUrl);
|
|
1264
|
+
const annotation = {
|
|
1265
|
+
"@context": "http://www.w3.org/ns/anno.jsonld",
|
|
1266
|
+
"type": "Annotation",
|
|
1267
|
+
"id": annotationId2,
|
|
1268
|
+
"motivation": "assessing",
|
|
1269
|
+
"creator": userId3(creatorUserId),
|
|
1270
|
+
"created": (/* @__PURE__ */ new Date()).toISOString(),
|
|
1271
|
+
"target": {
|
|
1272
|
+
type: "SpecificResource",
|
|
1273
|
+
source: resourceUri2,
|
|
1274
|
+
selector: [
|
|
1275
|
+
{
|
|
1276
|
+
type: "TextPositionSelector",
|
|
1277
|
+
start: assessment.start,
|
|
1278
|
+
end: assessment.end
|
|
1279
|
+
},
|
|
1280
|
+
{
|
|
1281
|
+
type: "TextQuoteSelector",
|
|
1282
|
+
exact: assessment.exact,
|
|
1283
|
+
...assessment.prefix && { prefix: assessment.prefix },
|
|
1284
|
+
...assessment.suffix && { suffix: assessment.suffix }
|
|
1285
|
+
}
|
|
1286
|
+
]
|
|
1287
|
+
},
|
|
1288
|
+
"body": {
|
|
1289
|
+
type: "TextualBody",
|
|
1290
|
+
value: assessment.assessment,
|
|
1291
|
+
format: "text/plain"
|
|
1292
|
+
}
|
|
1293
|
+
};
|
|
1294
|
+
await this.eventStore.appendEvent({
|
|
1295
|
+
type: "annotation.added",
|
|
1296
|
+
resourceId: resourceId2,
|
|
1297
|
+
userId: userId3(creatorUserId),
|
|
1298
|
+
version: 1,
|
|
1299
|
+
payload: { annotation }
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1302
|
+
};
|
|
1303
|
+
|
|
1304
|
+
// src/jobs/workers/tag-detection-worker.ts
|
|
1305
|
+
import { JobWorker as JobWorker4 } from "@semiont/jobs";
|
|
1306
|
+
import { generateAnnotationId as generateAnnotationId4 } from "@semiont/event-sourcing";
|
|
1307
|
+
import { resourceIdToURI as resourceIdToURI5 } from "@semiont/core";
|
|
1308
|
+
import { getTagSchema as getTagSchema2 } from "@semiont/ontology";
|
|
1309
|
+
import { userId as userId4 } from "@semiont/core";
|
|
1310
|
+
var TagDetectionWorker = class extends JobWorker4 {
|
|
1311
|
+
constructor(jobQueue, config, eventStore) {
|
|
1312
|
+
super(jobQueue);
|
|
1313
|
+
this.config = config;
|
|
1314
|
+
this.eventStore = eventStore;
|
|
1315
|
+
}
|
|
1316
|
+
isFirstProgress = true;
|
|
1317
|
+
getWorkerName() {
|
|
1318
|
+
return "TagDetectionWorker";
|
|
1319
|
+
}
|
|
1320
|
+
canProcessJob(job) {
|
|
1321
|
+
return job.metadata.type === "tag-detection";
|
|
1322
|
+
}
|
|
1323
|
+
async executeJob(job) {
|
|
1324
|
+
if (job.metadata.type !== "tag-detection") {
|
|
1325
|
+
throw new Error(`Invalid job type: ${job.metadata.type}`);
|
|
1326
|
+
}
|
|
1327
|
+
if (job.status !== "running") {
|
|
1328
|
+
throw new Error(`Job must be in running state to execute, got: ${job.status}`);
|
|
1329
|
+
}
|
|
1330
|
+
this.isFirstProgress = true;
|
|
1331
|
+
await this.processTagDetectionJob(job);
|
|
1332
|
+
}
|
|
1333
|
+
/**
|
|
1334
|
+
* Override updateJobProgress to emit events to Event Store
|
|
1335
|
+
*/
|
|
1336
|
+
async updateJobProgress(job) {
|
|
1337
|
+
await super.updateJobProgress(job);
|
|
1338
|
+
if (job.metadata.type !== "tag-detection") return;
|
|
1339
|
+
if (job.status !== "running") {
|
|
1340
|
+
return;
|
|
1341
|
+
}
|
|
1342
|
+
const tdJob = job;
|
|
1343
|
+
const baseEvent = {
|
|
1344
|
+
resourceId: tdJob.params.resourceId,
|
|
1345
|
+
userId: tdJob.metadata.userId,
|
|
1346
|
+
version: 1
|
|
1347
|
+
};
|
|
1348
|
+
const isComplete = tdJob.progress.percentage === 100;
|
|
1349
|
+
if (this.isFirstProgress) {
|
|
1350
|
+
this.isFirstProgress = false;
|
|
1351
|
+
await this.eventStore.appendEvent({
|
|
1352
|
+
type: "job.started",
|
|
1353
|
+
...baseEvent,
|
|
1354
|
+
payload: {
|
|
1355
|
+
jobId: tdJob.metadata.id,
|
|
1356
|
+
jobType: tdJob.metadata.type
|
|
1357
|
+
}
|
|
1358
|
+
});
|
|
1359
|
+
} else if (isComplete) {
|
|
1360
|
+
await this.eventStore.appendEvent({
|
|
1361
|
+
type: "job.completed",
|
|
1362
|
+
...baseEvent,
|
|
1363
|
+
payload: {
|
|
1364
|
+
jobId: tdJob.metadata.id,
|
|
1365
|
+
jobType: tdJob.metadata.type
|
|
1366
|
+
// Note: result would come from job.result, but that's handled by base class
|
|
1367
|
+
}
|
|
1368
|
+
});
|
|
1369
|
+
} else {
|
|
1370
|
+
await this.eventStore.appendEvent({
|
|
1371
|
+
type: "job.progress",
|
|
1372
|
+
...baseEvent,
|
|
1373
|
+
payload: {
|
|
1374
|
+
jobId: tdJob.metadata.id,
|
|
1375
|
+
jobType: tdJob.metadata.type,
|
|
1376
|
+
progress: tdJob.progress
|
|
1377
|
+
}
|
|
1378
|
+
});
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
async handleJobFailure(job, error) {
|
|
1382
|
+
await super.handleJobFailure(job, error);
|
|
1383
|
+
if (job.status === "failed" && job.metadata.type === "tag-detection") {
|
|
1384
|
+
const tdJob = job;
|
|
1385
|
+
await this.eventStore.appendEvent({
|
|
1386
|
+
type: "job.failed",
|
|
1387
|
+
resourceId: tdJob.params.resourceId,
|
|
1388
|
+
userId: tdJob.metadata.userId,
|
|
1389
|
+
version: 1,
|
|
1390
|
+
payload: {
|
|
1391
|
+
jobId: tdJob.metadata.id,
|
|
1392
|
+
jobType: tdJob.metadata.type,
|
|
1393
|
+
error: "Tag detection failed. Please try again later."
|
|
1394
|
+
}
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
async processTagDetectionJob(job) {
|
|
1399
|
+
console.log(`[TagDetectionWorker] Processing tag detection for resource ${job.params.resourceId} (job: ${job.metadata.id})`);
|
|
1400
|
+
const schema = getTagSchema2(job.params.schemaId);
|
|
1401
|
+
if (!schema) {
|
|
1402
|
+
throw new Error(`Invalid tag schema: ${job.params.schemaId}`);
|
|
1403
|
+
}
|
|
1404
|
+
for (const category of job.params.categories) {
|
|
1405
|
+
if (!schema.tags.some((t) => t.name === category)) {
|
|
1406
|
+
throw new Error(`Invalid category "${category}" for schema ${job.params.schemaId}`);
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
const resource = await ResourceContext.getResourceMetadata(job.params.resourceId, this.config);
|
|
1410
|
+
if (!resource) {
|
|
1411
|
+
throw new Error(`Resource ${job.params.resourceId} not found`);
|
|
1412
|
+
}
|
|
1413
|
+
let updatedJob = {
|
|
1414
|
+
...job,
|
|
1415
|
+
progress: {
|
|
1416
|
+
stage: "analyzing",
|
|
1417
|
+
percentage: 10,
|
|
1418
|
+
processedCategories: 0,
|
|
1419
|
+
totalCategories: job.params.categories.length,
|
|
1420
|
+
message: "Loading resource..."
|
|
1421
|
+
}
|
|
1422
|
+
};
|
|
1423
|
+
await this.updateJobProgress(updatedJob);
|
|
1424
|
+
const allTags = [];
|
|
1425
|
+
const byCategory = {};
|
|
1426
|
+
for (let i = 0; i < job.params.categories.length; i++) {
|
|
1427
|
+
const category = job.params.categories[i];
|
|
1428
|
+
updatedJob = {
|
|
1429
|
+
...updatedJob,
|
|
1430
|
+
progress: {
|
|
1431
|
+
stage: "analyzing",
|
|
1432
|
+
percentage: 10 + Math.floor(i / job.params.categories.length * 50),
|
|
1433
|
+
currentCategory: category,
|
|
1434
|
+
processedCategories: i + 1,
|
|
1435
|
+
totalCategories: job.params.categories.length,
|
|
1436
|
+
message: `Analyzing ${category}...`
|
|
1437
|
+
}
|
|
1438
|
+
};
|
|
1439
|
+
await this.updateJobProgress(updatedJob);
|
|
1440
|
+
const tags = await AnnotationDetection.detectTags(
|
|
1441
|
+
job.params.resourceId,
|
|
1442
|
+
this.config,
|
|
1443
|
+
job.params.schemaId,
|
|
1444
|
+
category
|
|
1445
|
+
);
|
|
1446
|
+
console.log(`[TagDetectionWorker] Found ${tags.length} tags for category "${category}"`);
|
|
1447
|
+
allTags.push(...tags);
|
|
1448
|
+
byCategory[category] = tags.length;
|
|
1449
|
+
}
|
|
1450
|
+
updatedJob = {
|
|
1451
|
+
...updatedJob,
|
|
1452
|
+
progress: {
|
|
1453
|
+
stage: "creating",
|
|
1454
|
+
percentage: 60,
|
|
1455
|
+
processedCategories: job.params.categories.length,
|
|
1456
|
+
totalCategories: job.params.categories.length,
|
|
1457
|
+
message: `Creating ${allTags.length} tag annotations...`
|
|
1458
|
+
}
|
|
1459
|
+
};
|
|
1460
|
+
await this.updateJobProgress(updatedJob);
|
|
1461
|
+
let created = 0;
|
|
1462
|
+
for (const tag of allTags) {
|
|
1463
|
+
try {
|
|
1464
|
+
await this.createTagAnnotation(job.params.resourceId, job.metadata.userId, job.params.schemaId, tag);
|
|
1465
|
+
created++;
|
|
1466
|
+
} catch (error) {
|
|
1467
|
+
console.error(`[TagDetectionWorker] Failed to create tag:`, error);
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
updatedJob = {
|
|
1471
|
+
...updatedJob,
|
|
1472
|
+
progress: {
|
|
1473
|
+
stage: "creating",
|
|
1474
|
+
percentage: 100,
|
|
1475
|
+
processedCategories: job.params.categories.length,
|
|
1476
|
+
totalCategories: job.params.categories.length,
|
|
1477
|
+
message: `Complete! Created ${created} tags`
|
|
1478
|
+
}
|
|
1479
|
+
};
|
|
1480
|
+
await this.updateJobProgress(updatedJob);
|
|
1481
|
+
console.log(`[TagDetectionWorker] \u2705 Created ${created}/${allTags.length} tags across ${job.params.categories.length} categories`);
|
|
1482
|
+
}
|
|
1483
|
+
async createTagAnnotation(resourceId2, userId_, schemaId, tag) {
|
|
1484
|
+
const backendUrl = this.config.services.backend?.publicURL;
|
|
1485
|
+
if (!backendUrl) {
|
|
1486
|
+
throw new Error("Backend publicURL not configured");
|
|
1487
|
+
}
|
|
1488
|
+
const resourceUri2 = resourceIdToURI5(resourceId2, backendUrl);
|
|
1489
|
+
const annotationId2 = generateAnnotationId4(backendUrl);
|
|
1490
|
+
const annotation = {
|
|
1491
|
+
"@context": "http://www.w3.org/ns/anno.jsonld",
|
|
1492
|
+
type: "Annotation",
|
|
1493
|
+
id: annotationId2,
|
|
1494
|
+
motivation: "tagging",
|
|
1495
|
+
target: {
|
|
1496
|
+
type: "SpecificResource",
|
|
1497
|
+
source: resourceUri2,
|
|
1498
|
+
selector: [
|
|
1499
|
+
{
|
|
1500
|
+
type: "TextPositionSelector",
|
|
1501
|
+
start: tag.start,
|
|
1502
|
+
end: tag.end
|
|
1503
|
+
},
|
|
1504
|
+
{
|
|
1505
|
+
type: "TextQuoteSelector",
|
|
1506
|
+
exact: tag.exact,
|
|
1507
|
+
prefix: tag.prefix || "",
|
|
1508
|
+
suffix: tag.suffix || ""
|
|
1509
|
+
}
|
|
1510
|
+
]
|
|
1511
|
+
},
|
|
1512
|
+
body: [
|
|
1513
|
+
{
|
|
1514
|
+
type: "TextualBody",
|
|
1515
|
+
value: tag.category,
|
|
1516
|
+
purpose: "tagging",
|
|
1517
|
+
format: "text/plain",
|
|
1518
|
+
language: "en"
|
|
1519
|
+
},
|
|
1520
|
+
{
|
|
1521
|
+
type: "TextualBody",
|
|
1522
|
+
value: schemaId,
|
|
1523
|
+
purpose: "classifying",
|
|
1524
|
+
format: "text/plain"
|
|
1525
|
+
}
|
|
1526
|
+
]
|
|
1527
|
+
};
|
|
1528
|
+
await this.eventStore.appendEvent({
|
|
1529
|
+
type: "annotation.added",
|
|
1530
|
+
resourceId: resourceId2,
|
|
1531
|
+
userId: userId4(userId_),
|
|
1532
|
+
version: 1,
|
|
1533
|
+
payload: {
|
|
1534
|
+
annotation
|
|
1535
|
+
}
|
|
1536
|
+
});
|
|
1537
|
+
console.log(`[TagDetectionWorker] Created tag annotation ${annotationId2} for "${tag.category}": "${tag.exact.substring(0, 50)}..."`);
|
|
1538
|
+
}
|
|
1539
|
+
};
|
|
1540
|
+
|
|
1541
|
+
// src/jobs/workers/reference-detection-worker.ts
|
|
1542
|
+
import { JobWorker as JobWorker5 } from "@semiont/jobs";
|
|
1543
|
+
import { generateAnnotationId as generateAnnotationId5 } from "@semiont/event-sourcing";
|
|
1544
|
+
import { resourceIdToURI as resourceIdToURI6 } from "@semiont/core";
|
|
1545
|
+
import {
|
|
1546
|
+
getPrimaryRepresentation as getPrimaryRepresentation4,
|
|
1547
|
+
decodeRepresentation as decodeRepresentation4,
|
|
1548
|
+
validateAndCorrectOffsets
|
|
1549
|
+
} from "@semiont/api-client";
|
|
1550
|
+
import { extractEntities } from "@semiont/inference";
|
|
1551
|
+
import { FilesystemRepresentationStore as FilesystemRepresentationStore4 } from "@semiont/content";
|
|
1552
|
+
var ReferenceDetectionWorker = class extends JobWorker5 {
|
|
1553
|
+
constructor(jobQueue, config, eventStore) {
|
|
1554
|
+
super(jobQueue);
|
|
1555
|
+
this.config = config;
|
|
1556
|
+
this.eventStore = eventStore;
|
|
1557
|
+
}
|
|
1558
|
+
getWorkerName() {
|
|
1559
|
+
return "ReferenceDetectionWorker";
|
|
1560
|
+
}
|
|
1561
|
+
canProcessJob(job) {
|
|
1562
|
+
return job.metadata.type === "detection";
|
|
1563
|
+
}
|
|
1564
|
+
async executeJob(job) {
|
|
1565
|
+
if (job.metadata.type !== "detection") {
|
|
1566
|
+
throw new Error(`Invalid job type: ${job.metadata.type}`);
|
|
1567
|
+
}
|
|
1568
|
+
if (job.status !== "running") {
|
|
1569
|
+
throw new Error(`Job must be in running state to execute, got: ${job.status}`);
|
|
1570
|
+
}
|
|
1571
|
+
await this.processDetectionJob(job);
|
|
1572
|
+
}
|
|
1573
|
+
/**
|
|
1574
|
+
* Detect entity references in resource using AI
|
|
1575
|
+
* Self-contained implementation for reference detection
|
|
1576
|
+
*
|
|
1577
|
+
* Public for testing charset handling - see entity-detection-charset.test.ts
|
|
1578
|
+
*/
|
|
1579
|
+
async detectReferences(resource, entityTypes, includeDescriptiveReferences = false) {
|
|
1580
|
+
console.log(`Detecting entities of types: ${entityTypes.join(", ")}${includeDescriptiveReferences ? " (including descriptive references)" : ""}`);
|
|
1581
|
+
const detectedAnnotations = [];
|
|
1582
|
+
const primaryRep = getPrimaryRepresentation4(resource);
|
|
1583
|
+
if (!primaryRep) return detectedAnnotations;
|
|
1584
|
+
const mediaType = primaryRep.mediaType;
|
|
1585
|
+
const baseMediaType = mediaType?.split(";")[0]?.trim() || "";
|
|
1586
|
+
if (baseMediaType === "text/plain" || baseMediaType === "text/markdown") {
|
|
1587
|
+
if (!primaryRep.checksum || !primaryRep.mediaType) return detectedAnnotations;
|
|
1588
|
+
const basePath = this.config.services.filesystem.path;
|
|
1589
|
+
const projectRoot = this.config._metadata?.projectRoot;
|
|
1590
|
+
const repStore = new FilesystemRepresentationStore4({ basePath }, projectRoot);
|
|
1591
|
+
const contentBuffer = await repStore.retrieve(primaryRep.checksum, primaryRep.mediaType);
|
|
1592
|
+
const content = decodeRepresentation4(contentBuffer, primaryRep.mediaType);
|
|
1593
|
+
const extractedEntities = await extractEntities(content, entityTypes, this.config, includeDescriptiveReferences);
|
|
1594
|
+
for (const entity of extractedEntities) {
|
|
1595
|
+
try {
|
|
1596
|
+
const validated = validateAndCorrectOffsets(
|
|
1597
|
+
content,
|
|
1598
|
+
entity.startOffset,
|
|
1599
|
+
entity.endOffset,
|
|
1600
|
+
entity.exact
|
|
1601
|
+
);
|
|
1602
|
+
const annotation = {
|
|
1603
|
+
annotation: {
|
|
1604
|
+
selector: {
|
|
1605
|
+
start: validated.start,
|
|
1606
|
+
end: validated.end,
|
|
1607
|
+
exact: validated.exact,
|
|
1608
|
+
prefix: validated.prefix,
|
|
1609
|
+
suffix: validated.suffix
|
|
1610
|
+
},
|
|
1611
|
+
entityTypes: [entity.entityType]
|
|
1612
|
+
}
|
|
1613
|
+
};
|
|
1614
|
+
detectedAnnotations.push(annotation);
|
|
1615
|
+
} catch (error) {
|
|
1616
|
+
console.warn(`[ReferenceDetectionWorker] Skipping invalid entity "${entity.exact}":`, error);
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
return detectedAnnotations;
|
|
1621
|
+
}
|
|
1622
|
+
async processDetectionJob(job) {
|
|
1623
|
+
console.log(`[ReferenceDetectionWorker] Processing detection for resource ${job.params.resourceId} (job: ${job.metadata.id})`);
|
|
1624
|
+
console.log(`[ReferenceDetectionWorker] \u{1F50D} Entity types: ${job.params.entityTypes.join(", ")}`);
|
|
1625
|
+
const resource = await ResourceContext.getResourceMetadata(job.params.resourceId, this.config);
|
|
1626
|
+
if (!resource) {
|
|
1627
|
+
throw new Error(`Resource ${job.params.resourceId} not found`);
|
|
1628
|
+
}
|
|
1629
|
+
let totalFound = 0;
|
|
1630
|
+
let totalEmitted = 0;
|
|
1631
|
+
let totalErrors = 0;
|
|
1632
|
+
let updatedJob = {
|
|
1633
|
+
...job,
|
|
1634
|
+
progress: {
|
|
1635
|
+
totalEntityTypes: job.params.entityTypes.length,
|
|
1636
|
+
processedEntityTypes: 0,
|
|
1637
|
+
entitiesFound: 0,
|
|
1638
|
+
entitiesEmitted: 0
|
|
1639
|
+
}
|
|
1640
|
+
};
|
|
1641
|
+
await this.updateJobProgress(updatedJob);
|
|
1642
|
+
for (let i = 0; i < job.params.entityTypes.length; i++) {
|
|
1643
|
+
const entityType = job.params.entityTypes[i];
|
|
1644
|
+
if (!entityType) continue;
|
|
1645
|
+
console.log(`[ReferenceDetectionWorker] \u{1F916} [${i + 1}/${job.params.entityTypes.length}] Detecting ${entityType}...`);
|
|
1646
|
+
const detectedAnnotations = await this.detectReferences(resource, [entityType], job.params.includeDescriptiveReferences);
|
|
1647
|
+
totalFound += detectedAnnotations.length;
|
|
1648
|
+
console.log(`[ReferenceDetectionWorker] \u2705 Found ${detectedAnnotations.length} ${entityType} entities`);
|
|
1649
|
+
for (let idx = 0; idx < detectedAnnotations.length; idx++) {
|
|
1650
|
+
const detected = detectedAnnotations[idx];
|
|
1651
|
+
if (!detected) {
|
|
1652
|
+
console.warn(`[ReferenceDetectionWorker] Skipping undefined entity at index ${idx}`);
|
|
1653
|
+
continue;
|
|
1654
|
+
}
|
|
1655
|
+
let referenceId;
|
|
1656
|
+
try {
|
|
1657
|
+
const backendUrl = this.config.services.backend?.publicURL;
|
|
1658
|
+
if (!backendUrl) {
|
|
1659
|
+
throw new Error("Backend publicURL not configured");
|
|
1660
|
+
}
|
|
1661
|
+
referenceId = generateAnnotationId5(backendUrl);
|
|
1662
|
+
} catch (error) {
|
|
1663
|
+
console.error(`[ReferenceDetectionWorker] Failed to generate annotation ID:`, error);
|
|
1664
|
+
throw new Error("Configuration error: Backend publicURL not set");
|
|
1665
|
+
}
|
|
1666
|
+
try {
|
|
1667
|
+
await this.eventStore.appendEvent({
|
|
1668
|
+
type: "annotation.added",
|
|
1669
|
+
resourceId: job.params.resourceId,
|
|
1670
|
+
userId: job.metadata.userId,
|
|
1671
|
+
version: 1,
|
|
1672
|
+
payload: {
|
|
1673
|
+
annotation: {
|
|
1674
|
+
"@context": "http://www.w3.org/ns/anno.jsonld",
|
|
1675
|
+
"type": "Annotation",
|
|
1676
|
+
id: referenceId,
|
|
1677
|
+
motivation: "linking",
|
|
1678
|
+
target: {
|
|
1679
|
+
source: resourceIdToURI6(job.params.resourceId, this.config.services.backend.publicURL),
|
|
1680
|
+
// Convert to full URI
|
|
1681
|
+
selector: [
|
|
1682
|
+
{
|
|
1683
|
+
type: "TextPositionSelector",
|
|
1684
|
+
start: detected.annotation.selector.start,
|
|
1685
|
+
end: detected.annotation.selector.end
|
|
1686
|
+
},
|
|
1687
|
+
{
|
|
1688
|
+
type: "TextQuoteSelector",
|
|
1689
|
+
exact: detected.annotation.selector.exact,
|
|
1690
|
+
...detected.annotation.selector.prefix && { prefix: detected.annotation.selector.prefix },
|
|
1691
|
+
...detected.annotation.selector.suffix && { suffix: detected.annotation.selector.suffix }
|
|
1692
|
+
}
|
|
1693
|
+
]
|
|
1694
|
+
},
|
|
1695
|
+
body: (detected.annotation.entityTypes || []).map((et) => ({
|
|
1696
|
+
type: "TextualBody",
|
|
1697
|
+
value: et,
|
|
1698
|
+
purpose: "tagging"
|
|
1699
|
+
})),
|
|
1700
|
+
modified: (/* @__PURE__ */ new Date()).toISOString()
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
});
|
|
1704
|
+
totalEmitted++;
|
|
1705
|
+
if ((idx + 1) % 10 === 0 || idx === detectedAnnotations.length - 1) {
|
|
1706
|
+
console.log(`[ReferenceDetectionWorker] \u{1F4E4} Emitted ${idx + 1}/${detectedAnnotations.length} events for ${entityType}`);
|
|
1707
|
+
}
|
|
1708
|
+
} catch (error) {
|
|
1709
|
+
totalErrors++;
|
|
1710
|
+
console.error(`[ReferenceDetectionWorker] \u274C Failed to emit event for ${referenceId}:`, error);
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
console.log(`[ReferenceDetectionWorker] \u2705 Completed ${entityType}: ${detectedAnnotations.length} found, ${detectedAnnotations.length - (totalErrors - (totalFound - totalEmitted))} emitted`);
|
|
1714
|
+
updatedJob = {
|
|
1715
|
+
...updatedJob,
|
|
1716
|
+
progress: {
|
|
1717
|
+
totalEntityTypes: job.params.entityTypes.length,
|
|
1718
|
+
processedEntityTypes: i + 1,
|
|
1719
|
+
currentEntityType: entityType,
|
|
1720
|
+
entitiesFound: totalFound,
|
|
1721
|
+
entitiesEmitted: totalEmitted
|
|
1722
|
+
}
|
|
1723
|
+
};
|
|
1724
|
+
await this.updateJobProgress(updatedJob);
|
|
1725
|
+
}
|
|
1726
|
+
console.log(`[ReferenceDetectionWorker] \u2705 Detection complete: ${totalFound} entities found, ${totalEmitted} events emitted, ${totalErrors} errors`);
|
|
1727
|
+
}
|
|
1728
|
+
async handleJobFailure(job, error) {
|
|
1729
|
+
await super.handleJobFailure(job, error);
|
|
1730
|
+
if (job.status === "failed" && job.metadata.type === "detection") {
|
|
1731
|
+
const detJob = job;
|
|
1732
|
+
await this.eventStore.appendEvent({
|
|
1733
|
+
type: "job.failed",
|
|
1734
|
+
resourceId: detJob.params.resourceId,
|
|
1735
|
+
userId: detJob.metadata.userId,
|
|
1736
|
+
version: 1,
|
|
1737
|
+
payload: {
|
|
1738
|
+
jobId: detJob.metadata.id,
|
|
1739
|
+
jobType: detJob.metadata.type,
|
|
1740
|
+
error: "Entity detection failed. Please try again later."
|
|
1741
|
+
}
|
|
1742
|
+
});
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
/**
|
|
1746
|
+
* Update job progress and emit events to Event Store
|
|
1747
|
+
* Overrides base class to also emit job progress events
|
|
1748
|
+
*/
|
|
1749
|
+
async updateJobProgress(job) {
|
|
1750
|
+
await super.updateJobProgress(job);
|
|
1751
|
+
if (job.metadata.type !== "detection") {
|
|
1752
|
+
return;
|
|
1753
|
+
}
|
|
1754
|
+
if (job.status !== "running") {
|
|
1755
|
+
return;
|
|
1756
|
+
}
|
|
1757
|
+
const detJob = job;
|
|
1758
|
+
const baseEvent = {
|
|
1759
|
+
resourceId: detJob.params.resourceId,
|
|
1760
|
+
userId: detJob.metadata.userId,
|
|
1761
|
+
version: 1
|
|
1762
|
+
};
|
|
1763
|
+
const isFirstUpdate = detJob.progress.processedEntityTypes === 0;
|
|
1764
|
+
const isFinalUpdate = detJob.progress.processedEntityTypes === detJob.progress.totalEntityTypes && detJob.progress.totalEntityTypes > 0;
|
|
1765
|
+
if (isFirstUpdate) {
|
|
1766
|
+
await this.eventStore.appendEvent({
|
|
1767
|
+
type: "job.started",
|
|
1768
|
+
...baseEvent,
|
|
1769
|
+
payload: {
|
|
1770
|
+
jobId: detJob.metadata.id,
|
|
1771
|
+
jobType: detJob.metadata.type,
|
|
1772
|
+
totalSteps: detJob.params.entityTypes.length
|
|
1773
|
+
}
|
|
1774
|
+
});
|
|
1775
|
+
} else if (isFinalUpdate) {
|
|
1776
|
+
await this.eventStore.appendEvent({
|
|
1777
|
+
type: "job.completed",
|
|
1778
|
+
...baseEvent,
|
|
1779
|
+
payload: {
|
|
1780
|
+
jobId: detJob.metadata.id,
|
|
1781
|
+
jobType: detJob.metadata.type,
|
|
1782
|
+
foundCount: detJob.progress.entitiesFound
|
|
1783
|
+
}
|
|
1784
|
+
});
|
|
1785
|
+
} else {
|
|
1786
|
+
const percentage = Math.round(detJob.progress.processedEntityTypes / detJob.progress.totalEntityTypes * 100);
|
|
1787
|
+
await this.eventStore.appendEvent({
|
|
1788
|
+
type: "job.progress",
|
|
1789
|
+
...baseEvent,
|
|
1790
|
+
payload: {
|
|
1791
|
+
jobId: detJob.metadata.id,
|
|
1792
|
+
jobType: detJob.metadata.type,
|
|
1793
|
+
percentage,
|
|
1794
|
+
currentStep: detJob.progress.currentEntityType,
|
|
1795
|
+
processedSteps: detJob.progress.processedEntityTypes,
|
|
1796
|
+
totalSteps: detJob.progress.totalEntityTypes,
|
|
1797
|
+
foundCount: detJob.progress.entitiesFound
|
|
1798
|
+
}
|
|
1799
|
+
});
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
};
|
|
1803
|
+
|
|
1804
|
+
// src/jobs/workers/generation-worker.ts
|
|
1805
|
+
import { JobWorker as JobWorker6 } from "@semiont/jobs";
|
|
1806
|
+
import { FilesystemRepresentationStore as FilesystemRepresentationStore5 } from "@semiont/content";
|
|
1807
|
+
import { generateResourceFromTopic } from "@semiont/inference";
|
|
1808
|
+
import {
|
|
1809
|
+
getTargetSelector as getTargetSelector2,
|
|
1810
|
+
getExactText,
|
|
1811
|
+
resourceUri,
|
|
1812
|
+
annotationUri
|
|
1813
|
+
} from "@semiont/api-client";
|
|
1814
|
+
import { getEntityTypes as getEntityTypes2 } from "@semiont/ontology";
|
|
1815
|
+
import {
|
|
1816
|
+
CREATION_METHODS,
|
|
1817
|
+
generateUuid,
|
|
1818
|
+
resourceId,
|
|
1819
|
+
annotationId
|
|
1820
|
+
} from "@semiont/core";
|
|
1821
|
+
var GenerationWorker = class extends JobWorker6 {
|
|
1822
|
+
constructor(jobQueue, config, eventStore) {
|
|
1823
|
+
super(jobQueue);
|
|
1824
|
+
this.config = config;
|
|
1825
|
+
this.eventStore = eventStore;
|
|
1826
|
+
}
|
|
1827
|
+
getWorkerName() {
|
|
1828
|
+
return "GenerationWorker";
|
|
1829
|
+
}
|
|
1830
|
+
canProcessJob(job) {
|
|
1831
|
+
return job.metadata.type === "generation";
|
|
1832
|
+
}
|
|
1833
|
+
async executeJob(job) {
|
|
1834
|
+
if (job.metadata.type !== "generation") {
|
|
1835
|
+
throw new Error(`Invalid job type: ${job.metadata.type}`);
|
|
1836
|
+
}
|
|
1837
|
+
if (job.status !== "running") {
|
|
1838
|
+
throw new Error(`Job must be in running state to execute, got: ${job.status}`);
|
|
1839
|
+
}
|
|
1840
|
+
await this.processGenerationJob(job);
|
|
1841
|
+
}
|
|
1842
|
+
async processGenerationJob(job) {
|
|
1843
|
+
console.log(`[GenerationWorker] Processing generation for reference ${job.params.referenceId} (job: ${job.metadata.id})`);
|
|
1844
|
+
const basePath = this.config.services.filesystem.path;
|
|
1845
|
+
const projectRoot = this.config._metadata?.projectRoot;
|
|
1846
|
+
const repStore = new FilesystemRepresentationStore5({ basePath }, projectRoot);
|
|
1847
|
+
let updatedJob = {
|
|
1848
|
+
...job,
|
|
1849
|
+
progress: {
|
|
1850
|
+
stage: "fetching",
|
|
1851
|
+
percentage: 20,
|
|
1852
|
+
message: "Fetching source resource..."
|
|
1853
|
+
}
|
|
1854
|
+
};
|
|
1855
|
+
console.log(`[GenerationWorker] \u{1F4E5} ${updatedJob.progress.message}`);
|
|
1856
|
+
await this.updateJobProgress(updatedJob);
|
|
1857
|
+
const { FilesystemViewStorage: FilesystemViewStorage3 } = await import("@semiont/event-sourcing");
|
|
1858
|
+
const viewStorage = new FilesystemViewStorage3(basePath, projectRoot);
|
|
1859
|
+
const view = await viewStorage.get(job.params.sourceResourceId);
|
|
1860
|
+
if (!view) {
|
|
1861
|
+
throw new Error(`Resource ${job.params.sourceResourceId} not found`);
|
|
1862
|
+
}
|
|
1863
|
+
const projection = view.annotations;
|
|
1864
|
+
const expectedAnnotationUri = `${this.config.services.backend.publicURL}/annotations/${job.params.referenceId}`;
|
|
1865
|
+
const annotation = projection.annotations.find(
|
|
1866
|
+
(a) => a.id === expectedAnnotationUri && a.motivation === "linking"
|
|
1867
|
+
);
|
|
1868
|
+
if (!annotation) {
|
|
1869
|
+
throw new Error(`Annotation ${job.params.referenceId} not found in resource ${job.params.sourceResourceId}`);
|
|
1870
|
+
}
|
|
1871
|
+
const sourceResource = await ResourceContext.getResourceMetadata(job.params.sourceResourceId, this.config);
|
|
1872
|
+
if (!sourceResource) {
|
|
1873
|
+
throw new Error(`Source resource ${job.params.sourceResourceId} not found`);
|
|
1874
|
+
}
|
|
1875
|
+
const targetSelector = getTargetSelector2(annotation.target);
|
|
1876
|
+
const resourceName = job.params.title || (targetSelector ? getExactText(targetSelector) : "") || "New Resource";
|
|
1877
|
+
console.log(`[GenerationWorker] Generating resource: "${resourceName}"`);
|
|
1878
|
+
if (!job.params.context) {
|
|
1879
|
+
throw new Error("Generation context is required but was not provided in job");
|
|
1880
|
+
}
|
|
1881
|
+
console.log(`[GenerationWorker] Using pre-fetched context: ${job.params.context.sourceContext?.before?.length || 0} chars before, ${job.params.context.sourceContext?.selected?.length || 0} chars selected, ${job.params.context.sourceContext?.after?.length || 0} chars after`);
|
|
1882
|
+
updatedJob = {
|
|
1883
|
+
...updatedJob,
|
|
1884
|
+
progress: {
|
|
1885
|
+
stage: "generating",
|
|
1886
|
+
percentage: 40,
|
|
1887
|
+
message: "Creating content with AI..."
|
|
1888
|
+
}
|
|
1889
|
+
};
|
|
1890
|
+
console.log(`[GenerationWorker] \u{1F916} ${updatedJob.progress.message}`);
|
|
1891
|
+
await this.updateJobProgress(updatedJob);
|
|
1892
|
+
const prompt = job.params.prompt || `Create a comprehensive resource about "${resourceName}"`;
|
|
1893
|
+
const annotationEntityTypes = getEntityTypes2({ body: annotation.body });
|
|
1894
|
+
const generatedContent = await generateResourceFromTopic(
|
|
1895
|
+
resourceName,
|
|
1896
|
+
job.params.entityTypes || annotationEntityTypes,
|
|
1897
|
+
this.config,
|
|
1898
|
+
prompt,
|
|
1899
|
+
job.params.language,
|
|
1900
|
+
job.params.context,
|
|
1901
|
+
// NEW - context from job (passed from modal)
|
|
1902
|
+
job.params.temperature,
|
|
1903
|
+
// NEW - from job
|
|
1904
|
+
job.params.maxTokens
|
|
1905
|
+
// NEW - from job
|
|
1906
|
+
);
|
|
1907
|
+
console.log(`[GenerationWorker] \u2705 Generated ${generatedContent.content.length} bytes of content`);
|
|
1908
|
+
updatedJob = {
|
|
1909
|
+
...updatedJob,
|
|
1910
|
+
progress: {
|
|
1911
|
+
stage: "generating",
|
|
1912
|
+
percentage: 70,
|
|
1913
|
+
message: "Content ready, creating resource..."
|
|
1914
|
+
}
|
|
1915
|
+
};
|
|
1916
|
+
await this.updateJobProgress(updatedJob);
|
|
1917
|
+
const rId = resourceId(generateUuid());
|
|
1918
|
+
updatedJob = {
|
|
1919
|
+
...updatedJob,
|
|
1920
|
+
progress: {
|
|
1921
|
+
stage: "creating",
|
|
1922
|
+
percentage: 85,
|
|
1923
|
+
message: "Saving resource..."
|
|
1924
|
+
}
|
|
1925
|
+
};
|
|
1926
|
+
console.log(`[GenerationWorker] \u{1F4BE} ${updatedJob.progress.message}`);
|
|
1927
|
+
await this.updateJobProgress(updatedJob);
|
|
1928
|
+
const storedRep = await repStore.store(Buffer.from(generatedContent.content), {
|
|
1929
|
+
mediaType: "text/markdown",
|
|
1930
|
+
rel: "original"
|
|
1931
|
+
});
|
|
1932
|
+
console.log(`[GenerationWorker] \u2705 Saved resource representation to filesystem: ${rId}`);
|
|
1933
|
+
await this.eventStore.appendEvent({
|
|
1934
|
+
type: "resource.created",
|
|
1935
|
+
resourceId: rId,
|
|
1936
|
+
userId: job.metadata.userId,
|
|
1937
|
+
version: 1,
|
|
1938
|
+
payload: {
|
|
1939
|
+
name: resourceName,
|
|
1940
|
+
format: "text/markdown",
|
|
1941
|
+
contentChecksum: storedRep.checksum,
|
|
1942
|
+
creationMethod: CREATION_METHODS.GENERATED,
|
|
1943
|
+
entityTypes: job.params.entityTypes || annotationEntityTypes,
|
|
1944
|
+
language: job.params.language,
|
|
1945
|
+
isDraft: true,
|
|
1946
|
+
generatedFrom: job.params.referenceId,
|
|
1947
|
+
generationPrompt: void 0
|
|
1948
|
+
// Could be added if we track the prompt
|
|
1949
|
+
}
|
|
1950
|
+
});
|
|
1951
|
+
console.log(`[GenerationWorker] Emitted resource.created event for ${rId}`);
|
|
1952
|
+
updatedJob = {
|
|
1953
|
+
...updatedJob,
|
|
1954
|
+
progress: {
|
|
1955
|
+
stage: "linking",
|
|
1956
|
+
percentage: 95,
|
|
1957
|
+
message: "Linking reference...",
|
|
1958
|
+
resultResourceId: rId
|
|
1959
|
+
// Store for job.completed event
|
|
1960
|
+
}
|
|
1961
|
+
};
|
|
1962
|
+
console.log(`[GenerationWorker] \u{1F517} ${updatedJob.progress.message}`);
|
|
1963
|
+
await this.updateJobProgress(updatedJob);
|
|
1964
|
+
const newResourceUri = resourceUri(`${this.config.services.backend.publicURL}/resources/${rId}`);
|
|
1965
|
+
const operations = [{
|
|
1966
|
+
op: "add",
|
|
1967
|
+
item: {
|
|
1968
|
+
type: "SpecificResource",
|
|
1969
|
+
source: newResourceUri,
|
|
1970
|
+
purpose: "linking"
|
|
1971
|
+
}
|
|
1972
|
+
}];
|
|
1973
|
+
const annotationIdSegment = job.params.referenceId.split("/").pop();
|
|
1974
|
+
await this.eventStore.appendEvent({
|
|
1975
|
+
type: "annotation.body.updated",
|
|
1976
|
+
resourceId: job.params.sourceResourceId,
|
|
1977
|
+
userId: job.metadata.userId,
|
|
1978
|
+
version: 1,
|
|
1979
|
+
payload: {
|
|
1980
|
+
annotationId: annotationId(annotationIdSegment),
|
|
1981
|
+
operations
|
|
1982
|
+
}
|
|
1983
|
+
});
|
|
1984
|
+
console.log(`[GenerationWorker] \u2705 Emitted annotation.body.updated event linking ${job.params.referenceId} \u2192 ${rId}`);
|
|
1985
|
+
updatedJob = {
|
|
1986
|
+
...updatedJob,
|
|
1987
|
+
progress: {
|
|
1988
|
+
stage: "linking",
|
|
1989
|
+
percentage: 100,
|
|
1990
|
+
message: "Complete!",
|
|
1991
|
+
resultResourceId: rId
|
|
1992
|
+
// Store for job.completed event
|
|
1993
|
+
}
|
|
1994
|
+
};
|
|
1995
|
+
await this.updateJobProgress(updatedJob);
|
|
1996
|
+
console.log(`[GenerationWorker] \u2705 Generation complete: created resource ${rId}`);
|
|
1997
|
+
}
|
|
1998
|
+
/**
|
|
1999
|
+
* Update job progress and emit events to Event Store
|
|
2000
|
+
* Overrides base class to also emit job progress events
|
|
2001
|
+
*/
|
|
2002
|
+
async updateJobProgress(job) {
|
|
2003
|
+
await super.updateJobProgress(job);
|
|
2004
|
+
if (job.metadata.type !== "generation") {
|
|
2005
|
+
return;
|
|
2006
|
+
}
|
|
2007
|
+
if (job.status !== "running") {
|
|
2008
|
+
return;
|
|
2009
|
+
}
|
|
2010
|
+
const genJob = job;
|
|
2011
|
+
const baseEvent = {
|
|
2012
|
+
resourceId: genJob.params.sourceResourceId,
|
|
2013
|
+
userId: genJob.metadata.userId,
|
|
2014
|
+
version: 1
|
|
2015
|
+
};
|
|
2016
|
+
if (genJob.progress.stage === "fetching" && genJob.progress.percentage === 20) {
|
|
2017
|
+
await this.eventStore.appendEvent({
|
|
2018
|
+
type: "job.started",
|
|
2019
|
+
...baseEvent,
|
|
2020
|
+
payload: {
|
|
2021
|
+
jobId: genJob.metadata.id,
|
|
2022
|
+
jobType: genJob.metadata.type,
|
|
2023
|
+
totalSteps: 5
|
|
2024
|
+
// fetching, generating, creating, linking, complete
|
|
2025
|
+
}
|
|
2026
|
+
});
|
|
2027
|
+
} else if (genJob.progress.stage === "linking" && genJob.progress.percentage === 100) {
|
|
2028
|
+
await this.eventStore.appendEvent({
|
|
2029
|
+
type: "job.completed",
|
|
2030
|
+
...baseEvent,
|
|
2031
|
+
payload: {
|
|
2032
|
+
jobId: genJob.metadata.id,
|
|
2033
|
+
jobType: genJob.metadata.type,
|
|
2034
|
+
resultResourceId: genJob.progress.resultResourceId,
|
|
2035
|
+
annotationUri: annotationUri(`${this.config.services.backend.publicURL}/annotations/${genJob.params.referenceId}`)
|
|
2036
|
+
}
|
|
2037
|
+
});
|
|
2038
|
+
} else {
|
|
2039
|
+
await this.eventStore.appendEvent({
|
|
2040
|
+
type: "job.progress",
|
|
2041
|
+
...baseEvent,
|
|
2042
|
+
payload: {
|
|
2043
|
+
jobId: genJob.metadata.id,
|
|
2044
|
+
jobType: genJob.metadata.type,
|
|
2045
|
+
currentStep: genJob.progress.stage,
|
|
2046
|
+
percentage: genJob.progress.percentage,
|
|
2047
|
+
message: genJob.progress.message
|
|
2048
|
+
}
|
|
2049
|
+
});
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
};
|
|
2053
|
+
|
|
704
2054
|
// src/index.ts
|
|
705
2055
|
var PACKAGE_NAME = "@semiont/make-meaning";
|
|
706
2056
|
var VERSION = "0.1.0";
|
|
707
2057
|
export {
|
|
708
2058
|
AnnotationContext,
|
|
709
2059
|
AnnotationDetection,
|
|
2060
|
+
AssessmentDetectionWorker,
|
|
2061
|
+
CommentDetectionWorker,
|
|
2062
|
+
GenerationWorker,
|
|
710
2063
|
GraphContext,
|
|
2064
|
+
HighlightDetectionWorker,
|
|
711
2065
|
PACKAGE_NAME,
|
|
2066
|
+
ReferenceDetectionWorker,
|
|
712
2067
|
ResourceContext,
|
|
2068
|
+
TagDetectionWorker,
|
|
713
2069
|
VERSION
|
|
714
2070
|
};
|
|
715
2071
|
//# sourceMappingURL=index.js.map
|