edgeflowjs 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -66
- package/dist/backends/index.d.ts +9 -2
- package/dist/backends/index.d.ts.map +1 -1
- package/dist/backends/index.js +13 -13
- package/dist/backends/index.js.map +1 -1
- package/dist/backends/onnx.d.ts +11 -4
- package/dist/backends/onnx.d.ts.map +1 -1
- package/dist/backends/onnx.js +97 -78
- package/dist/backends/onnx.js.map +1 -1
- package/dist/backends/transformers-adapter.d.ts +99 -0
- package/dist/backends/transformers-adapter.d.ts.map +1 -0
- package/dist/backends/transformers-adapter.js +171 -0
- package/dist/backends/transformers-adapter.js.map +1 -0
- package/dist/backends/webgpu.d.ts +7 -5
- package/dist/backends/webgpu.d.ts.map +1 -1
- package/dist/backends/webgpu.js +7 -5
- package/dist/backends/webgpu.js.map +1 -1
- package/dist/backends/webnn.d.ts +6 -5
- package/dist/backends/webnn.d.ts.map +1 -1
- package/dist/backends/webnn.js +6 -5
- package/dist/backends/webnn.js.map +1 -1
- package/dist/core/composer.d.ts +118 -0
- package/dist/core/composer.d.ts.map +1 -0
- package/dist/core/composer.js +163 -0
- package/dist/core/composer.js.map +1 -0
- package/dist/core/device-profiler.d.ts +75 -0
- package/dist/core/device-profiler.d.ts.map +1 -0
- package/dist/core/device-profiler.js +131 -0
- package/dist/core/device-profiler.js.map +1 -0
- package/dist/core/index.d.ts +4 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +8 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/memory.d.ts +22 -2
- package/dist/core/memory.d.ts.map +1 -1
- package/dist/core/memory.js +49 -13
- package/dist/core/memory.js.map +1 -1
- package/dist/core/plugin.d.ts +100 -0
- package/dist/core/plugin.d.ts.map +1 -0
- package/dist/core/plugin.js +106 -0
- package/dist/core/plugin.js.map +1 -0
- package/dist/core/runtime.d.ts +4 -0
- package/dist/core/runtime.d.ts.map +1 -1
- package/dist/core/runtime.js +18 -0
- package/dist/core/runtime.js.map +1 -1
- package/dist/core/scheduler.d.ts +17 -0
- package/dist/core/scheduler.d.ts.map +1 -1
- package/dist/core/scheduler.js +101 -3
- package/dist/core/scheduler.js.map +1 -1
- package/dist/core/types.d.ts +14 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/core/worker.d.ts +202 -0
- package/dist/core/worker.d.ts.map +1 -0
- package/dist/core/worker.js +477 -0
- package/dist/core/worker.js.map +1 -0
- package/dist/edgeflow.browser.js +9770 -4383
- package/dist/edgeflow.browser.js.map +4 -4
- package/dist/edgeflow.browser.min.js +435 -5
- package/dist/edgeflow.browser.min.js.map +4 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +28 -10
- package/dist/index.js.map +1 -1
- package/dist/pipelines/automatic-speech-recognition.d.ts +63 -0
- package/dist/pipelines/automatic-speech-recognition.d.ts.map +1 -0
- package/dist/pipelines/automatic-speech-recognition.js +269 -0
- package/dist/pipelines/automatic-speech-recognition.js.map +1 -0
- package/dist/pipelines/base.d.ts +6 -1
- package/dist/pipelines/base.d.ts.map +1 -1
- package/dist/pipelines/base.js +12 -2
- package/dist/pipelines/base.js.map +1 -1
- package/dist/pipelines/feature-extraction.d.ts +5 -40
- package/dist/pipelines/feature-extraction.d.ts.map +1 -1
- package/dist/pipelines/feature-extraction.js +44 -63
- package/dist/pipelines/feature-extraction.js.map +1 -1
- package/dist/pipelines/image-classification.d.ts +4 -36
- package/dist/pipelines/image-classification.d.ts.map +1 -1
- package/dist/pipelines/image-classification.js +22 -60
- package/dist/pipelines/image-classification.js.map +1 -1
- package/dist/pipelines/image-segmentation.d.ts +221 -0
- package/dist/pipelines/image-segmentation.d.ts.map +1 -0
- package/dist/pipelines/image-segmentation.js +535 -0
- package/dist/pipelines/image-segmentation.js.map +1 -0
- package/dist/pipelines/index.d.ts +18 -0
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +51 -2
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pipelines/object-detection.d.ts +44 -0
- package/dist/pipelines/object-detection.d.ts.map +1 -0
- package/dist/pipelines/object-detection.js +218 -0
- package/dist/pipelines/object-detection.js.map +1 -0
- package/dist/pipelines/question-answering.d.ts +41 -0
- package/dist/pipelines/question-answering.d.ts.map +1 -0
- package/dist/pipelines/question-answering.js +164 -0
- package/dist/pipelines/question-answering.js.map +1 -0
- package/dist/pipelines/text-classification.d.ts +3 -39
- package/dist/pipelines/text-classification.d.ts.map +1 -1
- package/dist/pipelines/text-classification.js +29 -67
- package/dist/pipelines/text-classification.js.map +1 -1
- package/dist/pipelines/text-generation.d.ts +281 -0
- package/dist/pipelines/text-generation.d.ts.map +1 -0
- package/dist/pipelines/text-generation.js +766 -0
- package/dist/pipelines/text-generation.js.map +1 -0
- package/dist/pipelines/zero-shot-classification.d.ts +45 -0
- package/dist/pipelines/zero-shot-classification.d.ts.map +1 -0
- package/dist/pipelines/zero-shot-classification.js +140 -0
- package/dist/pipelines/zero-shot-classification.js.map +1 -0
- package/dist/tools/benchmark.d.ts +92 -0
- package/dist/tools/benchmark.d.ts.map +1 -0
- package/dist/tools/benchmark.js +213 -0
- package/dist/tools/benchmark.js.map +1 -0
- package/dist/tools/debugger.d.ts +258 -0
- package/dist/tools/debugger.d.ts.map +1 -0
- package/dist/tools/debugger.js +624 -0
- package/dist/tools/debugger.js.map +1 -0
- package/dist/tools/index.d.ts +8 -0
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +16 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/monitor.d.ts +284 -0
- package/dist/tools/monitor.d.ts.map +1 -0
- package/dist/tools/monitor.js +921 -0
- package/dist/tools/monitor.js.map +1 -0
- package/dist/tools/quantization.d.ts +235 -0
- package/dist/tools/quantization.d.ts.map +1 -0
- package/dist/tools/quantization.js +830 -0
- package/dist/tools/quantization.js.map +1 -0
- package/dist/utils/hub.d.ts +162 -0
- package/dist/utils/hub.d.ts.map +1 -0
- package/dist/utils/hub.js +311 -0
- package/dist/utils/hub.js.map +1 -0
- package/dist/utils/index.d.ts +3 -1
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +5 -1
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/model-loader.d.ts.map +1 -1
- package/dist/utils/model-loader.js +106 -30
- package/dist/utils/model-loader.js.map +1 -1
- package/dist/utils/offline.d.ts +147 -0
- package/dist/utils/offline.d.ts.map +1 -0
- package/dist/utils/offline.js +405 -0
- package/dist/utils/offline.js.map +1 -0
- package/dist/utils/preprocessor.d.ts +82 -6
- package/dist/utils/preprocessor.d.ts.map +1 -1
- package/dist/utils/preprocessor.js +278 -21
- package/dist/utils/preprocessor.js.map +1 -1
- package/dist/utils/tokenizer.d.ts +197 -72
- package/dist/utils/tokenizer.d.ts.map +1 -1
- package/dist/utils/tokenizer.js +558 -274
- package/dist/utils/tokenizer.js.map +1 -1
- package/package.json +26 -11
package/dist/pipelines/index.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* edgeFlow.js - Pipeline Exports
|
|
3
3
|
*/
|
|
4
|
+
import { getPluginPipeline } from '../core/plugin.js';
|
|
5
|
+
import { registerAllBackends } from '../backends/index.js';
|
|
4
6
|
// Base
|
|
5
7
|
export { BasePipeline, registerPipeline, getPipelineFactory, SENTIMENT_LABELS, EMOTION_LABELS, IMAGENET_LABELS, } from './base.js';
|
|
6
8
|
// Text Classification
|
|
@@ -9,10 +11,28 @@ export { TextClassificationPipeline, SentimentAnalysisPipeline, createTextClassi
|
|
|
9
11
|
export { FeatureExtractionPipeline, createFeatureExtractionPipeline, } from './feature-extraction.js';
|
|
10
12
|
// Image Classification
|
|
11
13
|
export { ImageClassificationPipeline, createImageClassificationPipeline, } from './image-classification.js';
|
|
14
|
+
// Text Generation
|
|
15
|
+
export { TextGenerationPipeline, createTextGenerationPipeline, } from './text-generation.js';
|
|
16
|
+
// Object Detection
|
|
17
|
+
export { ObjectDetectionPipeline, createObjectDetectionPipeline, COCO_LABELS, } from './object-detection.js';
|
|
18
|
+
// Automatic Speech Recognition
|
|
19
|
+
export { AutomaticSpeechRecognitionPipeline, createASRPipeline, } from './automatic-speech-recognition.js';
|
|
20
|
+
// Zero-shot Classification
|
|
21
|
+
export { ZeroShotClassificationPipeline, createZeroShotClassificationPipeline, } from './zero-shot-classification.js';
|
|
22
|
+
// Question Answering
|
|
23
|
+
export { QuestionAnsweringPipeline, createQuestionAnsweringPipeline, } from './question-answering.js';
|
|
24
|
+
// Image Segmentation
|
|
25
|
+
export { ImageSegmentationPipeline, createImageSegmentationPipeline, } from './image-segmentation.js';
|
|
12
26
|
// Import pipeline classes
|
|
13
27
|
import { TextClassificationPipeline, SentimentAnalysisPipeline } from './text-classification.js';
|
|
14
28
|
import { FeatureExtractionPipeline } from './feature-extraction.js';
|
|
15
29
|
import { ImageClassificationPipeline } from './image-classification.js';
|
|
30
|
+
import { TextGenerationPipeline } from './text-generation.js';
|
|
31
|
+
import { ObjectDetectionPipeline } from './object-detection.js';
|
|
32
|
+
import { AutomaticSpeechRecognitionPipeline } from './automatic-speech-recognition.js';
|
|
33
|
+
import { ZeroShotClassificationPipeline } from './zero-shot-classification.js';
|
|
34
|
+
import { QuestionAnsweringPipeline } from './question-answering.js';
|
|
35
|
+
import { ImageSegmentationPipeline } from './image-segmentation.js';
|
|
16
36
|
/**
|
|
17
37
|
* Create a pipeline for a specific task
|
|
18
38
|
*
|
|
@@ -29,6 +49,9 @@ import { ImageClassificationPipeline } from './image-classification.js';
|
|
|
29
49
|
* ```
|
|
30
50
|
*/
|
|
31
51
|
export async function pipeline(task, options) {
|
|
52
|
+
// Guarantee backends are registered before any model loads.
|
|
53
|
+
// registerAllBackends() is synchronous and idempotent (safe to call repeatedly).
|
|
54
|
+
registerAllBackends();
|
|
32
55
|
const config = {
|
|
33
56
|
task: task,
|
|
34
57
|
model: options?.model ?? 'default',
|
|
@@ -50,8 +73,34 @@ export async function pipeline(task, options) {
|
|
|
50
73
|
case 'image-classification':
|
|
51
74
|
pipelineInstance = new ImageClassificationPipeline(config, options?.labels);
|
|
52
75
|
break;
|
|
53
|
-
|
|
54
|
-
|
|
76
|
+
case 'text-generation':
|
|
77
|
+
pipelineInstance = new TextGenerationPipeline(config);
|
|
78
|
+
break;
|
|
79
|
+
case 'object-detection':
|
|
80
|
+
pipelineInstance = new ObjectDetectionPipeline(config, options?.labels);
|
|
81
|
+
break;
|
|
82
|
+
case 'automatic-speech-recognition':
|
|
83
|
+
pipelineInstance = new AutomaticSpeechRecognitionPipeline(config);
|
|
84
|
+
break;
|
|
85
|
+
case 'zero-shot-classification':
|
|
86
|
+
pipelineInstance = new ZeroShotClassificationPipeline(config);
|
|
87
|
+
break;
|
|
88
|
+
case 'question-answering':
|
|
89
|
+
pipelineInstance = new QuestionAnsweringPipeline(config);
|
|
90
|
+
break;
|
|
91
|
+
case 'image-segmentation':
|
|
92
|
+
pipelineInstance = new ImageSegmentationPipeline(config);
|
|
93
|
+
break;
|
|
94
|
+
default: {
|
|
95
|
+
// Check if a plugin provides this pipeline task
|
|
96
|
+
const pluginEntry = getPluginPipeline(task);
|
|
97
|
+
if (pluginEntry) {
|
|
98
|
+
pipelineInstance = pluginEntry.factory(config);
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
throw new Error(`Unknown pipeline task: "${task}". ` +
|
|
102
|
+
`Register a plugin with registerPlugin() to add custom pipeline tasks.`);
|
|
103
|
+
}
|
|
55
104
|
}
|
|
56
105
|
// Initialize the pipeline
|
|
57
106
|
await pipelineInstance.initialize();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipelines/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,OAAO;AACP,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,eAAe,GAMhB,MAAM,WAAW,CAAC;AAEnB,sBAAsB;AACtB,OAAO,EACL,0BAA0B,EAC1B,yBAAyB,EACzB,gCAAgC,EAChC,+BAA+B,GAEhC,MAAM,0BAA0B,CAAC;AAElC,qBAAqB;AACrB,OAAO,EACL,yBAAyB,EACzB,+BAA+B,GAEhC,MAAM,yBAAyB,CAAC;AAEjC,uBAAuB;AACvB,OAAO,EACL,2BAA2B,EAC3B,iCAAiC,GAGlC,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipelines/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO;AACP,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,eAAe,GAMhB,MAAM,WAAW,CAAC;AAEnB,sBAAsB;AACtB,OAAO,EACL,0BAA0B,EAC1B,yBAAyB,EACzB,gCAAgC,EAChC,+BAA+B,GAEhC,MAAM,0BAA0B,CAAC;AAElC,qBAAqB;AACrB,OAAO,EACL,yBAAyB,EACzB,+BAA+B,GAEhC,MAAM,yBAAyB,CAAC;AAEjC,uBAAuB;AACvB,OAAO,EACL,2BAA2B,EAC3B,iCAAiC,GAGlC,MAAM,2BAA2B,CAAC;AAEnC,kBAAkB;AAClB,OAAO,EACL,sBAAsB,EACtB,4BAA4B,GAQ7B,MAAM,sBAAsB,CAAC;AAE9B,mBAAmB;AACnB,OAAO,EACL,uBAAuB,EACvB,6BAA6B,EAC7B,WAAW,GAIZ,MAAM,uBAAuB,CAAC;AAE/B,+BAA+B;AAC/B,OAAO,EACL,kCAAkC,EAClC,iBAAiB,GAKlB,MAAM,mCAAmC,CAAC;AAE3C,2BAA2B;AAC3B,OAAO,EACL,8BAA8B,EAC9B,oCAAoC,GAGrC,MAAM,+BAA+B,CAAC;AAEvC,qBAAqB;AACrB,OAAO,EACL,yBAAyB,EACzB,+BAA+B,GAIhC,MAAM,yBAAyB,CAAC;AAEjC,qBAAqB;AACrB,OAAO,EACL,yBAAyB,EACzB,+BAA+B,GAMhC,MAAM,yBAAyB,CAAC;AAsCjC,0BAA0B;AAC1B,OAAO,EAAE,0BAA0B,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACjG,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AACxE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,kCAAkC,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAEpE;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAO,EACP,OAAgC;IAEhC,4DAA4D;IAC5D,iFAAiF;IACjF,mBAAmB,EAAE,CAAC;IAEtB,MAAM,MAAM,GAAmB;QAC7B,IAAI,EAAE,IAAoB;QAC1B,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,SAAS;QAClC,OAAO,EAAE,OAAO,EAAE,OAAO;QACzB,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,IAAI;QAC7B,YAAY,EAAE,OAAO,EAAE,YAAY;KACpC,CAAC;IAIF,IAAI,gBAA8B,CAAC;IAEnC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,qBAAqB;YACxB,gBAAgB,GAAG,IAAI,0BAA0B,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3E,MAAM;QACR,KAAK,oBAAoB;YACvB,gBAAgB,GAAG,IAAI,yBAAyB,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM;QACR,KAAK,oBAAoB;YACvB,gBAAgB,GAAG,IAAI,yBAAyB,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM;QACR,KAAK,sBAAsB;YACzB,gBAAgB,GAAG,IAAI,2BAA2B,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC5E,MAAM;QACR,KAAK,iBAAiB;YACpB,gBAAgB,GAAG,IAAI,sBAAsB,CAAC,MAAM,CAAC,CAAC;YACtD,MAAM;QACR,KAAK,kBAAkB;YACrB,gBAAgB,GAAG,IAAI,uBAAuB,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACxE,MAAM;QACR,KAAK,8BAA8B;YACjC,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,MAAM,CAAC,CAAC;YAClE,MAAM;QACR,KAAK,0BAA0B;YAC7B,gBAAgB,GAAG,IAAI,8BAA8B,CAAC,MAAM,CAAC,CAAC;YAC9D,MAAM;QACR,KAAK,oBAAoB;YACvB,gBAAgB,GAAG,IAAI,yBAAyB,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM;QACR,KAAK,oBAAoB;YACvB,gBAAgB,GAAG,IAAI,yBAAyB,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM;QACR,OAAO,CAAC,CAAC,CAAC;YACR,gDAAgD;YAChD,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,WAAW,EAAE,CAAC;gBAChB,gBAAgB,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/C,MAAM;YACR,CAAC;YACD,MAAM,IAAI,KAAK,CACb,2BAA2B,IAAI,KAAK;gBACpC,uEAAuE,CACxE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,MAAM,gBAAgB,CAAC,UAAU,EAAE,CAAC;IAEpC,OAAO,gBAAsC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAQ,EACR,OAAgC;IAEhC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAC3C,CAAC;IAEF,MAAM,MAAM,GAAsD,EAAE,CAAC;IAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACvB,MAAM,CAAC,IAAiB,CAAC,GAAG,SAAS,CAAC,CAAC,CAA+B,CAAC;IACzE,CAAC;IAED,OAAO,MAAkD,CAAC;AAC5D,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* edgeFlow.js - Object Detection Pipeline
|
|
3
|
+
*
|
|
4
|
+
* Detect objects in images with bounding boxes and class labels.
|
|
5
|
+
*/
|
|
6
|
+
import { BasePipeline, ObjectDetectionResult } from './base.js';
|
|
7
|
+
import { EdgeFlowTensor } from '../core/tensor.js';
|
|
8
|
+
import { PipelineConfig, PipelineOptions } from '../core/types.js';
|
|
9
|
+
import { type ImageInput } from '../utils/preprocessor.js';
|
|
10
|
+
export interface ObjectDetectionOptions extends PipelineOptions {
|
|
11
|
+
threshold?: number;
|
|
12
|
+
topK?: number;
|
|
13
|
+
nms?: boolean;
|
|
14
|
+
iouThreshold?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface BoundingBox {
|
|
17
|
+
x: number;
|
|
18
|
+
y: number;
|
|
19
|
+
width: number;
|
|
20
|
+
height: number;
|
|
21
|
+
}
|
|
22
|
+
export interface Detection extends ObjectDetectionResult {
|
|
23
|
+
classId: number;
|
|
24
|
+
boxNormalized: BoundingBox;
|
|
25
|
+
}
|
|
26
|
+
export declare const COCO_LABELS: string[];
|
|
27
|
+
export declare class ObjectDetectionPipeline extends BasePipeline<ImageInput | ImageInput[], Detection[]> {
|
|
28
|
+
private preprocessor;
|
|
29
|
+
private onnxModel;
|
|
30
|
+
private labels;
|
|
31
|
+
private modelUrl;
|
|
32
|
+
constructor(config?: PipelineConfig, labels?: string[]);
|
|
33
|
+
initialize(): Promise<void>;
|
|
34
|
+
setLabels(labels: string[]): void;
|
|
35
|
+
run(input: ImageInput | ImageInput[], options?: ObjectDetectionOptions): Promise<Detection[]>;
|
|
36
|
+
protected preprocess(input: ImageInput | ImageInput[]): Promise<EdgeFlowTensor[]>;
|
|
37
|
+
private runModelInference;
|
|
38
|
+
protected postprocess(outputs: EdgeFlowTensor[], options?: PipelineOptions): Promise<Detection[]>;
|
|
39
|
+
private parseDetections;
|
|
40
|
+
private nonMaxSuppression;
|
|
41
|
+
private computeIoU;
|
|
42
|
+
}
|
|
43
|
+
export declare function createObjectDetectionPipeline(config?: PipelineConfig, labels?: string[]): ObjectDetectionPipeline;
|
|
44
|
+
//# sourceMappingURL=object-detection.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"object-detection.d.ts","sourceRoot":"","sources":["../../src/pipelines/object-detection.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAoB,MAAM,WAAW,CAAC;AAClF,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAe,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAQ9E,MAAM,WAAW,sBAAuB,SAAQ,eAAe;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,SAAU,SAAQ,qBAAqB;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,WAAW,CAAC;CAC5B;AAcD,eAAO,MAAM,WAAW,UAavB,CAAC;AAMF,qBAAa,uBAAwB,SAAQ,YAAY,CAAC,UAAU,GAAG,UAAU,EAAE,EAAE,SAAS,EAAE,CAAC;IAC/F,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE;IAiBvC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAS1C,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAIlB,GAAG,CAChB,KAAK,EAAE,UAAU,GAAG,UAAU,EAAE,EAChC,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,SAAS,EAAE,CAAC;cAOP,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,EAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;YAezE,iBAAiB;cAKf,WAAW,CACzB,OAAO,EAAE,cAAc,EAAE,EACzB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,SAAS,EAAE,CAAC;IAwBvB,OAAO,CAAC,eAAe;IAuFvB,OAAO,CAAC,iBAAiB;IAgCzB,OAAO,CAAC,UAAU;CAenB;AAMD,wBAAgB,6BAA6B,CAC3C,MAAM,CAAC,EAAE,cAAc,EACvB,MAAM,CAAC,EAAE,MAAM,EAAE,GAChB,uBAAuB,CAEzB"}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* edgeFlow.js - Object Detection Pipeline
|
|
3
|
+
*
|
|
4
|
+
* Detect objects in images with bounding boxes and class labels.
|
|
5
|
+
*/
|
|
6
|
+
import { BasePipeline, registerPipeline } from './base.js';
|
|
7
|
+
import { EdgeFlowTensor } from '../core/tensor.js';
|
|
8
|
+
import { ImagePreprocessor } from '../utils/preprocessor.js';
|
|
9
|
+
import { loadModelData } from '../utils/model-loader.js';
|
|
10
|
+
import { loadModelFromBuffer, runInference } from '../core/runtime.js';
|
|
11
|
+
// ============================================================================
|
|
12
|
+
// Default Model (YOLOS-tiny, quantized)
|
|
13
|
+
// ============================================================================
|
|
14
|
+
const DEFAULT_MODELS = {
|
|
15
|
+
model: 'https://huggingface.co/Xenova/yolos-tiny/resolve/main/onnx/model_quantized.onnx',
|
|
16
|
+
};
|
|
17
|
+
// ============================================================================
|
|
18
|
+
// COCO Labels
|
|
19
|
+
// ============================================================================
|
|
20
|
+
export const COCO_LABELS = [
|
|
21
|
+
'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck',
|
|
22
|
+
'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench',
|
|
23
|
+
'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra',
|
|
24
|
+
'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
|
|
25
|
+
'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove',
|
|
26
|
+
'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup',
|
|
27
|
+
'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange',
|
|
28
|
+
'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
|
|
29
|
+
'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse',
|
|
30
|
+
'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink',
|
|
31
|
+
'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier',
|
|
32
|
+
'toothbrush'
|
|
33
|
+
];
|
|
34
|
+
// ============================================================================
|
|
35
|
+
// Object Detection Pipeline
|
|
36
|
+
// ============================================================================
|
|
37
|
+
export class ObjectDetectionPipeline extends BasePipeline {
|
|
38
|
+
preprocessor;
|
|
39
|
+
onnxModel = null;
|
|
40
|
+
labels;
|
|
41
|
+
modelUrl;
|
|
42
|
+
constructor(config, labels) {
|
|
43
|
+
super(config ?? {
|
|
44
|
+
task: 'object-detection',
|
|
45
|
+
model: 'default',
|
|
46
|
+
});
|
|
47
|
+
this.labels = labels ?? COCO_LABELS;
|
|
48
|
+
this.modelUrl = (config?.model && config.model !== 'default') ? config.model : DEFAULT_MODELS.model;
|
|
49
|
+
this.preprocessor = new ImagePreprocessor({
|
|
50
|
+
width: 640,
|
|
51
|
+
height: 640,
|
|
52
|
+
mean: [0.485, 0.456, 0.406],
|
|
53
|
+
std: [0.229, 0.224, 0.225],
|
|
54
|
+
channelFormat: 'CHW',
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
async initialize() {
|
|
58
|
+
await super.initialize();
|
|
59
|
+
if (!this.onnxModel) {
|
|
60
|
+
const modelData = await loadModelData(this.modelUrl, { cache: this.config.cache ?? true });
|
|
61
|
+
this.onnxModel = await loadModelFromBuffer(modelData);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
setLabels(labels) {
|
|
65
|
+
this.labels = labels;
|
|
66
|
+
}
|
|
67
|
+
async run(input, options) {
|
|
68
|
+
await this.initialize();
|
|
69
|
+
const tensorInputs = await this.preprocess(input);
|
|
70
|
+
const outputs = await this.runModelInference(tensorInputs);
|
|
71
|
+
return this.postprocess(outputs, options);
|
|
72
|
+
}
|
|
73
|
+
async preprocess(input) {
|
|
74
|
+
const inputs = Array.isArray(input) ? input : [input];
|
|
75
|
+
if (inputs.length === 1) {
|
|
76
|
+
const tensor = await this.preprocessor.process(inputs[0]);
|
|
77
|
+
return [new EdgeFlowTensor(tensor.toFloat32Array(), [1, ...tensor.shape], 'float32')];
|
|
78
|
+
}
|
|
79
|
+
return [await this.preprocessor.processBatch(inputs)];
|
|
80
|
+
}
|
|
81
|
+
async runModelInference(inputs) {
|
|
82
|
+
const outputs = await runInference(this.onnxModel, inputs);
|
|
83
|
+
return outputs;
|
|
84
|
+
}
|
|
85
|
+
async postprocess(outputs, options) {
|
|
86
|
+
const opts = options ?? {};
|
|
87
|
+
const threshold = opts.threshold ?? 0.5;
|
|
88
|
+
const topK = opts.topK ?? 100;
|
|
89
|
+
const nms = opts.nms ?? true;
|
|
90
|
+
const iouThreshold = opts.iouThreshold ?? 0.5;
|
|
91
|
+
if (!outputs[0]) {
|
|
92
|
+
return [];
|
|
93
|
+
}
|
|
94
|
+
const outputData = outputs[0].toFloat32Array();
|
|
95
|
+
const shape = [...outputs[0].shape];
|
|
96
|
+
const detections = this.parseDetections(outputData, shape, threshold);
|
|
97
|
+
let filtered = nms ? this.nonMaxSuppression(detections, iouThreshold) : detections;
|
|
98
|
+
filtered.sort((a, b) => b.score - a.score);
|
|
99
|
+
filtered = filtered.slice(0, topK);
|
|
100
|
+
return filtered;
|
|
101
|
+
}
|
|
102
|
+
parseDetections(data, shape, threshold) {
|
|
103
|
+
const detections = [];
|
|
104
|
+
const numBoxes = shape[1] ?? 0;
|
|
105
|
+
const boxSize = shape[2] ?? 0;
|
|
106
|
+
if (boxSize >= 5) {
|
|
107
|
+
const numClasses = boxSize - 5;
|
|
108
|
+
for (let i = 0; i < numBoxes; i++) {
|
|
109
|
+
const offset = i * boxSize;
|
|
110
|
+
const objectness = data[offset + 4] ?? 0;
|
|
111
|
+
if (objectness < threshold)
|
|
112
|
+
continue;
|
|
113
|
+
let maxClassScore = 0;
|
|
114
|
+
let maxClassIdx = 0;
|
|
115
|
+
for (let c = 0; c < numClasses; c++) {
|
|
116
|
+
const score = data[offset + 5 + c] ?? 0;
|
|
117
|
+
if (score > maxClassScore) {
|
|
118
|
+
maxClassScore = score;
|
|
119
|
+
maxClassIdx = c;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
const confidence = objectness * maxClassScore;
|
|
123
|
+
if (confidence < threshold)
|
|
124
|
+
continue;
|
|
125
|
+
const x = data[offset] ?? 0;
|
|
126
|
+
const y = data[offset + 1] ?? 0;
|
|
127
|
+
const w = data[offset + 2] ?? 0;
|
|
128
|
+
const h = data[offset + 3] ?? 0;
|
|
129
|
+
detections.push({
|
|
130
|
+
label: this.labels[maxClassIdx] ?? `class_${maxClassIdx}`,
|
|
131
|
+
score: confidence,
|
|
132
|
+
classId: maxClassIdx,
|
|
133
|
+
box: {
|
|
134
|
+
x: Math.max(0, x - w / 2),
|
|
135
|
+
y: Math.max(0, y - h / 2),
|
|
136
|
+
width: w,
|
|
137
|
+
height: h,
|
|
138
|
+
},
|
|
139
|
+
boxNormalized: {
|
|
140
|
+
x: Math.max(0, x - w / 2),
|
|
141
|
+
y: Math.max(0, y - h / 2),
|
|
142
|
+
width: w,
|
|
143
|
+
height: h,
|
|
144
|
+
},
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
else if (boxSize === 4) {
|
|
149
|
+
for (let i = 0; i < numBoxes; i++) {
|
|
150
|
+
const offset = i * boxSize;
|
|
151
|
+
const x1 = data[offset] ?? 0;
|
|
152
|
+
const y1 = data[offset + 1] ?? 0;
|
|
153
|
+
const x2 = data[offset + 2] ?? 0;
|
|
154
|
+
const y2 = data[offset + 3] ?? 0;
|
|
155
|
+
detections.push({
|
|
156
|
+
label: this.labels[0] ?? 'object',
|
|
157
|
+
score: 1.0,
|
|
158
|
+
classId: 0,
|
|
159
|
+
box: {
|
|
160
|
+
x: x1,
|
|
161
|
+
y: y1,
|
|
162
|
+
width: x2 - x1,
|
|
163
|
+
height: y2 - y1,
|
|
164
|
+
},
|
|
165
|
+
boxNormalized: {
|
|
166
|
+
x: x1,
|
|
167
|
+
y: y1,
|
|
168
|
+
width: x2 - x1,
|
|
169
|
+
height: y2 - y1,
|
|
170
|
+
},
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return detections;
|
|
175
|
+
}
|
|
176
|
+
nonMaxSuppression(detections, iouThreshold) {
|
|
177
|
+
if (detections.length === 0)
|
|
178
|
+
return [];
|
|
179
|
+
const sorted = [...detections].sort((a, b) => b.score - a.score);
|
|
180
|
+
const selected = [];
|
|
181
|
+
const active = new Array(sorted.length).fill(true);
|
|
182
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
183
|
+
if (!active[i])
|
|
184
|
+
continue;
|
|
185
|
+
const current = sorted[i];
|
|
186
|
+
selected.push(current);
|
|
187
|
+
for (let j = i + 1; j < sorted.length; j++) {
|
|
188
|
+
if (!active[j])
|
|
189
|
+
continue;
|
|
190
|
+
const other = sorted[j];
|
|
191
|
+
if (current.classId !== other.classId)
|
|
192
|
+
continue;
|
|
193
|
+
const iou = this.computeIoU(current.box, other.box);
|
|
194
|
+
if (iou > iouThreshold) {
|
|
195
|
+
active[j] = false;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return selected;
|
|
200
|
+
}
|
|
201
|
+
computeIoU(a, b) {
|
|
202
|
+
const xOverlap = Math.max(0, Math.min(a.x + a.width, b.x + b.width) - Math.max(a.x, b.x));
|
|
203
|
+
const yOverlap = Math.max(0, Math.min(a.y + a.height, b.y + b.height) - Math.max(a.y, b.y));
|
|
204
|
+
const intersection = xOverlap * yOverlap;
|
|
205
|
+
const aArea = a.width * a.height;
|
|
206
|
+
const bArea = b.width * b.height;
|
|
207
|
+
const union = aArea + bArea - intersection;
|
|
208
|
+
return union > 0 ? intersection / union : 0;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
// ============================================================================
|
|
212
|
+
// Factory
|
|
213
|
+
// ============================================================================
|
|
214
|
+
export function createObjectDetectionPipeline(config, labels) {
|
|
215
|
+
return new ObjectDetectionPipeline(config, labels);
|
|
216
|
+
}
|
|
217
|
+
registerPipeline('object-detection', (config) => new ObjectDetectionPipeline(config));
|
|
218
|
+
//# sourceMappingURL=object-detection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"object-detection.js","sourceRoot":"","sources":["../../src/pipelines/object-detection.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,YAAY,EAAyB,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAClF,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAmB,MAAM,0BAA0B,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAyBvE,+EAA+E;AAC/E,wCAAwC;AACxC,+EAA+E;AAE/E,MAAM,cAAc,GAAG;IACrB,KAAK,EAAE,iFAAiF;CACzF,CAAC;AAEF,+EAA+E;AAC/E,cAAc;AACd,+EAA+E;AAE/E,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO;IAC7E,MAAM,EAAE,eAAe,EAAE,cAAc,EAAE,WAAW,EAAE,eAAe,EAAE,OAAO;IAC9E,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO;IAC1E,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS;IAC1E,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,gBAAgB;IAC5E,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK;IACzE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ;IACzE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IAC3E,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO;IACxE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAC1E,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY;IAC/E,YAAY;CACb,CAAC;AAEF,+EAA+E;AAC/E,4BAA4B;AAC5B,+EAA+E;AAE/E,MAAM,OAAO,uBAAwB,SAAQ,YAAoD;IACvF,YAAY,CAAoB;IAChC,SAAS,GAAuB,IAAI,CAAC;IACrC,MAAM,CAAW;IACjB,QAAQ,CAAS;IAEzB,YAAY,MAAuB,EAAE,MAAiB;QACpD,KAAK,CAAC,MAAM,IAAI;YACd,IAAI,EAAE,kBAAkB;YACxB,KAAK,EAAE,SAAS;SACjB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,WAAW,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;QACpG,IAAI,CAAC,YAAY,GAAG,IAAI,iBAAiB,CAAC;YACxC,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,GAAG;YACX,IAAI,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;YAC3B,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;YAC1B,aAAa,EAAE,KAAK;SACrB,CAAC,CAAC;IACL,CAAC;IAEQ,KAAK,CAAC,UAAU;QACvB,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;YAC3F,IAAI,CAAC,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,SAAS,CAAC,MAAgB;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEQ,KAAK,CAAC,GAAG,CAChB,KAAgC,EAChC,OAAgC;QAEhC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAC3D,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,KAAgC;QACzD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAEtD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;YAC3D,OAAO,CAAC,IAAI,cAAc,CACxB,MAAM,CAAC,cAAc,EAAE,EACvB,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,EACpB,SAAS,CACV,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,MAAwB;QACtD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,SAAU,EAAE,MAAM,CAAC,CAAC;QAC5D,OAAO,OAA2B,CAAC;IACrC,CAAC;IAES,KAAK,CAAC,WAAW,CACzB,OAAyB,EACzB,OAAyB;QAEzB,MAAM,IAAI,GAAG,OAAiC,IAAI,EAAE,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC;QAC7B,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,GAAG,CAAC;QAE9C,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAChB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAa,CAAC;QAEhD,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QAEtE,IAAI,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QAEnF,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC3C,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAEnC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,eAAe,CACrB,IAAkB,EAClB,KAAe,EACf,SAAiB;QAEjB,MAAM,UAAU,GAAgB,EAAE,CAAC;QAEnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAE9B,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;YACjB,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,CAAC;YAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC;gBAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBAEzC,IAAI,UAAU,GAAG,SAAS;oBAAE,SAAS;gBAErC,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,IAAI,WAAW,GAAG,CAAC,CAAC;gBAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;oBACxC,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;wBAC1B,aAAa,GAAG,KAAK,CAAC;wBACtB,WAAW,GAAG,CAAC,CAAC;oBAClB,CAAC;gBACH,CAAC;gBAED,MAAM,UAAU,GAAG,UAAU,GAAG,aAAa,CAAC;gBAC9C,IAAI,UAAU,GAAG,SAAS;oBAAE,SAAS;gBAErC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBAEhC,UAAU,CAAC,IAAI,CAAC;oBACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,SAAS,WAAW,EAAE;oBACzD,KAAK,EAAE,UAAU;oBACjB,OAAO,EAAE,WAAW;oBACpB,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBACzB,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBACzB,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;qBACV;oBACD,aAAa,EAAE;wBACb,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBACzB,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBACzB,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;qBACV;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC;gBAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBAEjC,UAAU,CAAC,IAAI,CAAC;oBACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,QAAQ;oBACjC,KAAK,EAAE,GAAG;oBACV,OAAO,EAAE,CAAC;oBACV,GAAG,EAAE;wBACH,CAAC,EAAE,EAAE;wBACL,CAAC,EAAE,EAAE;wBACL,KAAK,EAAE,EAAE,GAAG,EAAE;wBACd,MAAM,EAAE,EAAE,GAAG,EAAE;qBAChB;oBACD,aAAa,EAAE;wBACb,CAAC,EAAE,EAAE;wBACL,CAAC,EAAE,EAAE;wBACL,KAAK,EAAE,EAAE,GAAG,EAAE;wBACd,MAAM,EAAE,EAAE,GAAG,EAAE;qBAChB;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,iBAAiB,CACvB,UAAuB,EACvB,YAAoB;QAEpB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAgB,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBAAE,SAAS;YAEzB,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;oBAAE,SAAS;gBAEzB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;gBACzB,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO;oBAAE,SAAS;gBAEhD,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,GAAG,GAAG,YAAY,EAAE,CAAC;oBACvB,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,UAAU,CAAC,CAAc,EAAE,CAAc;QAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EACzB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAC5D,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EACzB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAC9D,CAAC;QAEF,MAAM,YAAY,GAAG,QAAQ,GAAG,QAAQ,CAAC;QACzC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;QACjC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;QACjC,MAAM,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,CAAC;QAE3C,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;CACF;AAED,+EAA+E;AAC/E,UAAU;AACV,+EAA+E;AAE/E,MAAM,UAAU,6BAA6B,CAC3C,MAAuB,EACvB,MAAiB;IAEjB,OAAO,IAAI,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC;AAED,gBAAgB,CAAC,kBAAkB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* edgeFlow.js - Question Answering Pipeline
|
|
3
|
+
*
|
|
4
|
+
* Extract answers from context given a question using real ONNX QA models.
|
|
5
|
+
*/
|
|
6
|
+
import { BasePipeline, PipelineResult } from './base.js';
|
|
7
|
+
import { EdgeFlowTensor } from '../core/tensor.js';
|
|
8
|
+
import { PipelineConfig, PipelineOptions } from '../core/types.js';
|
|
9
|
+
import { Tokenizer } from '../utils/tokenizer.js';
|
|
10
|
+
export interface QAInput {
|
|
11
|
+
question: string;
|
|
12
|
+
context: string;
|
|
13
|
+
}
|
|
14
|
+
export interface QuestionAnsweringOptions extends PipelineOptions {
|
|
15
|
+
maxAnswerLength?: number;
|
|
16
|
+
maxQuestionLength?: number;
|
|
17
|
+
topK?: number;
|
|
18
|
+
threshold?: number;
|
|
19
|
+
handleImpossible?: boolean;
|
|
20
|
+
}
|
|
21
|
+
export interface QuestionAnsweringResult extends PipelineResult {
|
|
22
|
+
answer: string;
|
|
23
|
+
score: number;
|
|
24
|
+
start: number;
|
|
25
|
+
end: number;
|
|
26
|
+
}
|
|
27
|
+
export declare class QuestionAnsweringPipeline extends BasePipeline<QAInput | QAInput[], QuestionAnsweringResult | QuestionAnsweringResult[]> {
|
|
28
|
+
private tokenizer;
|
|
29
|
+
private onnxModel;
|
|
30
|
+
private modelUrl;
|
|
31
|
+
private tokenizerUrl;
|
|
32
|
+
constructor(config?: PipelineConfig);
|
|
33
|
+
initialize(): Promise<void>;
|
|
34
|
+
setTokenizer(tokenizer: Tokenizer): void;
|
|
35
|
+
run(input: QAInput | QAInput[], options?: QuestionAnsweringOptions): Promise<QuestionAnsweringResult | QuestionAnsweringResult[]>;
|
|
36
|
+
private answerQuestion;
|
|
37
|
+
protected preprocess(input: QAInput | QAInput[]): Promise<EdgeFlowTensor[]>;
|
|
38
|
+
protected postprocess(outputs: EdgeFlowTensor[], _options?: PipelineOptions): Promise<QuestionAnsweringResult | QuestionAnsweringResult[]>;
|
|
39
|
+
}
|
|
40
|
+
export declare function createQuestionAnsweringPipeline(config?: PipelineConfig): QuestionAnsweringPipeline;
|
|
41
|
+
//# sourceMappingURL=question-answering.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"question-answering.d.ts","sourceRoot":"","sources":["../../src/pipelines/question-answering.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,YAAY,EAAE,cAAc,EAAoB,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAW,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,eAAe,EAAe,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAiBlD,MAAM,WAAW,OAAO;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAyB,SAAQ,eAAe;IAC/D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,uBAAwB,SAAQ,cAAc;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAMD,qBAAa,yBAA0B,SAAQ,YAAY,CACzD,OAAO,GAAG,OAAO,EAAE,EACnB,uBAAuB,GAAG,uBAAuB,EAAE,CACpD;IACC,OAAO,CAAC,SAAS,CAA0B;IAC3C,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,YAAY,CAAS;gBAEjB,MAAM,CAAC,EAAE,cAAc;IASpB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAa1C,YAAY,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAIzB,GAAG,CAChB,KAAK,EAAE,OAAO,GAAG,OAAO,EAAE,EAC1B,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,uBAAuB,GAAG,uBAAuB,EAAE,CAAC;YAWjD,cAAc;cAqFZ,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;cAyBjE,WAAW,CACzB,OAAO,EAAE,cAAc,EAAE,EACzB,QAAQ,CAAC,EAAE,eAAe,GACzB,OAAO,CAAC,uBAAuB,GAAG,uBAAuB,EAAE,CAAC;CAkChE;AAMD,wBAAgB,+BAA+B,CAC7C,MAAM,CAAC,EAAE,cAAc,GACtB,yBAAyB,CAE3B"}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* edgeFlow.js - Question Answering Pipeline
|
|
3
|
+
*
|
|
4
|
+
* Extract answers from context given a question using real ONNX QA models.
|
|
5
|
+
*/
|
|
6
|
+
import { BasePipeline, registerPipeline } from './base.js';
|
|
7
|
+
import { EdgeFlowTensor, softmax } from '../core/tensor.js';
|
|
8
|
+
import { Tokenizer } from '../utils/tokenizer.js';
|
|
9
|
+
import { loadModelData } from '../utils/model-loader.js';
|
|
10
|
+
import { loadModelFromBuffer, runInferenceNamed } from '../core/runtime.js';
|
|
11
|
+
// ============================================================================
|
|
12
|
+
// Default Model (DistilBERT fine-tuned on SQuAD)
|
|
13
|
+
// ============================================================================
|
|
14
|
+
const DEFAULT_MODELS = {
|
|
15
|
+
model: 'https://huggingface.co/Xenova/distilbert-base-cased-distilled-squad/resolve/main/onnx/model_quantized.onnx',
|
|
16
|
+
tokenizer: 'https://huggingface.co/Xenova/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json',
|
|
17
|
+
};
|
|
18
|
+
// ============================================================================
|
|
19
|
+
// Question Answering Pipeline
|
|
20
|
+
// ============================================================================
|
|
21
|
+
export class QuestionAnsweringPipeline extends BasePipeline {
|
|
22
|
+
tokenizer = null;
|
|
23
|
+
onnxModel = null;
|
|
24
|
+
modelUrl;
|
|
25
|
+
tokenizerUrl;
|
|
26
|
+
constructor(config) {
|
|
27
|
+
super(config ?? {
|
|
28
|
+
task: 'question-answering',
|
|
29
|
+
model: 'default',
|
|
30
|
+
});
|
|
31
|
+
this.modelUrl = (config?.model && config.model !== 'default') ? config.model : DEFAULT_MODELS.model;
|
|
32
|
+
this.tokenizerUrl = DEFAULT_MODELS.tokenizer;
|
|
33
|
+
}
|
|
34
|
+
async initialize() {
|
|
35
|
+
await super.initialize();
|
|
36
|
+
if (!this.tokenizer) {
|
|
37
|
+
this.tokenizer = await Tokenizer.fromUrl(this.tokenizerUrl);
|
|
38
|
+
}
|
|
39
|
+
if (!this.onnxModel) {
|
|
40
|
+
const modelData = await loadModelData(this.modelUrl, { cache: this.config.cache ?? true });
|
|
41
|
+
this.onnxModel = await loadModelFromBuffer(modelData);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
setTokenizer(tokenizer) {
|
|
45
|
+
this.tokenizer = tokenizer;
|
|
46
|
+
}
|
|
47
|
+
async run(input, options) {
|
|
48
|
+
await this.initialize();
|
|
49
|
+
const inputs = Array.isArray(input) ? input : [input];
|
|
50
|
+
const results = await Promise.all(inputs.map(i => this.answerQuestion(i, options ?? {})));
|
|
51
|
+
return Array.isArray(input) ? results : results[0];
|
|
52
|
+
}
|
|
53
|
+
async answerQuestion(input, options) {
|
|
54
|
+
const startTime = performance.now();
|
|
55
|
+
const { question, context } = input;
|
|
56
|
+
const maxAnswerLength = options.maxAnswerLength ?? 30;
|
|
57
|
+
// No padding — QA runs one example at a time and padding wastes compute
|
|
58
|
+
const encoded = this.tokenizer.encode(question, {
|
|
59
|
+
textPair: context,
|
|
60
|
+
addSpecialTokens: true,
|
|
61
|
+
maxLength: 512,
|
|
62
|
+
truncation: true,
|
|
63
|
+
padding: 'do_not_pad',
|
|
64
|
+
returnAttentionMask: true,
|
|
65
|
+
returnTokenTypeIds: true,
|
|
66
|
+
});
|
|
67
|
+
const seqLen = encoded.inputIds.length;
|
|
68
|
+
const inputIds = new EdgeFlowTensor(BigInt64Array.from(encoded.inputIds.map(id => BigInt(id))), [1, seqLen], 'int64');
|
|
69
|
+
const attentionMask = new EdgeFlowTensor(BigInt64Array.from(encoded.attentionMask.map(m => BigInt(m))), [1, seqLen], 'int64');
|
|
70
|
+
const namedInputs = new Map();
|
|
71
|
+
namedInputs.set('input_ids', inputIds);
|
|
72
|
+
namedInputs.set('attention_mask', attentionMask);
|
|
73
|
+
const outputs = await runInferenceNamed(this.onnxModel, namedInputs);
|
|
74
|
+
if (outputs.length < 2) {
|
|
75
|
+
return { answer: '', score: 0, start: 0, end: 0, processingTime: performance.now() - startTime };
|
|
76
|
+
}
|
|
77
|
+
const startLogits = outputs[0].toFloat32Array();
|
|
78
|
+
const endLogits = outputs[1].toFloat32Array();
|
|
79
|
+
const startProbs = softmax(new EdgeFlowTensor(new Float32Array(startLogits), [seqLen], 'float32')).toFloat32Array();
|
|
80
|
+
const endProbs = softmax(new EdgeFlowTensor(new Float32Array(endLogits), [seqLen], 'float32')).toFloat32Array();
|
|
81
|
+
// Constrain answer span to the context portion only (tokenTypeIds === 1).
|
|
82
|
+
// tokenTypeIds: 0 = question tokens ([CLS], question, [SEP]), 1 = context tokens.
|
|
83
|
+
const typeIds = encoded.tokenTypeIds ?? new Array(seqLen).fill(1);
|
|
84
|
+
// Find where context starts (first index with typeId === 1)
|
|
85
|
+
const contextStart = typeIds.findIndex(t => t === 1);
|
|
86
|
+
const spanStart = contextStart >= 0 ? contextStart : 0;
|
|
87
|
+
const spanEnd = seqLen - 1; // last non-padding position
|
|
88
|
+
let bestStartIdx = spanStart;
|
|
89
|
+
let bestEndIdx = spanStart;
|
|
90
|
+
let bestScore = -Infinity;
|
|
91
|
+
for (let s = spanStart; s <= spanEnd; s++) {
|
|
92
|
+
for (let e = s; e < Math.min(s + maxAnswerLength, spanEnd + 1); e++) {
|
|
93
|
+
const score = (startProbs[s] ?? 0) * (endProbs[e] ?? 0);
|
|
94
|
+
if (score > bestScore) {
|
|
95
|
+
bestScore = score;
|
|
96
|
+
bestStartIdx = s;
|
|
97
|
+
bestEndIdx = e;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// Decode the answer span directly from token IDs in the context portion
|
|
102
|
+
const answerTokenIds = encoded.inputIds.slice(bestStartIdx, bestEndIdx + 1);
|
|
103
|
+
const answer = this.tokenizer.decode(answerTokenIds, true);
|
|
104
|
+
return {
|
|
105
|
+
answer: answer || '',
|
|
106
|
+
score: Math.max(0, bestScore),
|
|
107
|
+
start: bestStartIdx,
|
|
108
|
+
end: bestEndIdx,
|
|
109
|
+
processingTime: performance.now() - startTime,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
async preprocess(input) {
|
|
113
|
+
const qaInput = Array.isArray(input) ? input[0] : input;
|
|
114
|
+
const encoded = this.tokenizer.encode(qaInput.question, {
|
|
115
|
+
textPair: qaInput.context,
|
|
116
|
+
addSpecialTokens: true,
|
|
117
|
+
maxLength: 512,
|
|
118
|
+
truncation: true,
|
|
119
|
+
returnAttentionMask: true,
|
|
120
|
+
returnTokenTypeIds: true,
|
|
121
|
+
});
|
|
122
|
+
return [
|
|
123
|
+
new EdgeFlowTensor(BigInt64Array.from(encoded.inputIds.map(id => BigInt(id))), [1, encoded.inputIds.length], 'int64'),
|
|
124
|
+
new EdgeFlowTensor(BigInt64Array.from(encoded.attentionMask.map(m => BigInt(m))), [1, encoded.attentionMask.length], 'int64'),
|
|
125
|
+
];
|
|
126
|
+
}
|
|
127
|
+
async postprocess(outputs, _options) {
|
|
128
|
+
if (outputs.length < 2) {
|
|
129
|
+
return { answer: '', score: 0, start: 0, end: 0 };
|
|
130
|
+
}
|
|
131
|
+
const startLogits = outputs[0].toFloat32Array();
|
|
132
|
+
const endLogits = outputs[1].toFloat32Array();
|
|
133
|
+
const seqLen = startLogits.length;
|
|
134
|
+
const startProbs = softmax(new EdgeFlowTensor(startLogits, [seqLen], 'float32')).toFloat32Array();
|
|
135
|
+
const endProbs = softmax(new EdgeFlowTensor(endLogits, [seqLen], 'float32')).toFloat32Array();
|
|
136
|
+
let bestStart = 0;
|
|
137
|
+
let bestEnd = 0;
|
|
138
|
+
let bestScore = 0;
|
|
139
|
+
for (let start = 0; start < seqLen; start++) {
|
|
140
|
+
for (let end = start; end < Math.min(start + 30, seqLen); end++) {
|
|
141
|
+
const score = (startProbs[start] ?? 0) * (endProbs[end] ?? 0);
|
|
142
|
+
if (score > bestScore) {
|
|
143
|
+
bestScore = score;
|
|
144
|
+
bestStart = start;
|
|
145
|
+
bestEnd = end;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
answer: '',
|
|
151
|
+
score: bestScore,
|
|
152
|
+
start: bestStart,
|
|
153
|
+
end: bestEnd,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
// ============================================================================
|
|
158
|
+
// Factory
|
|
159
|
+
// ============================================================================
|
|
160
|
+
export function createQuestionAnsweringPipeline(config) {
|
|
161
|
+
return new QuestionAnsweringPipeline(config);
|
|
162
|
+
}
|
|
163
|
+
registerPipeline('question-answering', (config) => new QuestionAnsweringPipeline(config));
|
|
164
|
+
//# sourceMappingURL=question-answering.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"question-answering.js","sourceRoot":"","sources":["../../src/pipelines/question-answering.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,YAAY,EAAkB,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5D,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE5E,+EAA+E;AAC/E,iDAAiD;AACjD,+EAA+E;AAE/E,MAAM,cAAc,GAAG;IACrB,KAAK,EAAE,4GAA4G;IACnH,SAAS,EAAE,iGAAiG;CAC7G,CAAC;AA0BF,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E,MAAM,OAAO,yBAA0B,SAAQ,YAG9C;IACS,SAAS,GAAqB,IAAI,CAAC;IACnC,SAAS,GAAuB,IAAI,CAAC;IACrC,QAAQ,CAAS;IACjB,YAAY,CAAS;IAE7B,YAAY,MAAuB;QACjC,KAAK,CAAC,MAAM,IAAI;YACd,IAAI,EAAE,oBAAoB;YAC1B,KAAK,EAAE,SAAS;SACjB,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;QACpG,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC,SAAS,CAAC;IAC/C,CAAC;IAEQ,KAAK,CAAC,UAAU;QACvB,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;YAC3F,IAAI,CAAC,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,YAAY,CAAC,SAAoB;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAEQ,KAAK,CAAC,GAAG,CAChB,KAA0B,EAC1B,OAAkC;QAElC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAExB,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,CACvD,CAAC;QAEF,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;IACtD,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,KAAc,EACd,OAAiC;QAEjC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;QACpC,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC;QAEtD,wEAAwE;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC/C,QAAQ,EAAE,OAAO;YACjB,gBAAgB,EAAE,IAAI;YACtB,SAAS,EAAE,GAAG;YACd,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,YAAY;YACrB,mBAAmB,EAAE,IAAI;YACzB,kBAAkB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,cAAc,CACjC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAC1D,CAAC,CAAC,EAAE,MAAM,CAAC,EACX,OAAO,CACR,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,cAAc,CACtC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAC7D,CAAC,CAAC,EAAE,MAAM,CAAC,EACX,OAAO,CACR,CAAC;QAEF,MAAM,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;QACtD,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QACvC,WAAW,CAAC,GAAG,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;QAEjD,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,SAAU,EAAE,WAAW,CAAC,CAAC;QAEtE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,cAAc,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;QACnG,CAAC;QAED,MAAM,WAAW,GAAI,OAAO,CAAC,CAAC,CAAoB,CAAC,cAAc,EAAE,CAAC;QACpE,MAAM,SAAS,GAAI,OAAO,CAAC,CAAC,CAAoB,CAAC,cAAc,EAAE,CAAC;QAElE,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,cAAc,CAAC,IAAI,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;QACpH,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,cAAc,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;QAEhH,0EAA0E;QAC1E,kFAAkF;QAClF,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClE,4DAA4D;QAC5D,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,4BAA4B;QAExD,IAAI,YAAY,GAAG,SAAS,CAAC;QAC7B,IAAI,UAAU,GAAG,SAAS,CAAC;QAC3B,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;QAE1B,KAAK,IAAI,CAAC,GAAG,SAAS,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,eAAe,EAAE,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpE,MAAM,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACxD,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;oBACtB,SAAS,GAAG,KAAK,CAAC;oBAClB,YAAY,GAAG,CAAC,CAAC;oBACjB,UAAU,GAAG,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,wEAAwE;QACxE,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;QAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QAE5D,OAAO;YACL,MAAM,EAAE,MAAM,IAAI,EAAE;YACpB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC;YAC7B,KAAK,EAAE,YAAY;YACnB,GAAG,EAAE,UAAU;YACf,cAAc,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;SAC9C,CAAC;IACJ,CAAC;IAGS,KAAK,CAAC,UAAU,CAAC,KAA0B;QACnD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE;YACvD,QAAQ,EAAE,OAAO,CAAC,OAAO;YACzB,gBAAgB,EAAE,IAAI;YACtB,SAAS,EAAE,GAAG;YACd,UAAU,EAAE,IAAI;YAChB,mBAAmB,EAAE,IAAI;YACzB,kBAAkB,EAAE,IAAI;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,IAAI,cAAc,CAChB,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAC1D,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAC5B,OAAO,CACR;YACD,IAAI,cAAc,CAChB,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAC7D,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EACjC,OAAO,CACR;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,WAAW,CACzB,OAAyB,EACzB,QAA0B;QAE1B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,cAAc,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,cAAc,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;QAElC,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,cAAc,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;QAClG,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;QAE9F,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAC5C,KAAK,IAAI,GAAG,GAAG,KAAK,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gBAChE,MAAM,KAAK,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC9D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;oBACtB,SAAS,GAAG,KAAK,CAAC;oBAClB,SAAS,GAAG,KAAK,CAAC;oBAClB,OAAO,GAAG,GAAG,CAAC;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,SAAS;YAChB,GAAG,EAAE,OAAO;SACb,CAAC;IACJ,CAAC;CACF;AAED,+EAA+E;AAC/E,UAAU;AACV,+EAA+E;AAE/E,MAAM,UAAU,+BAA+B,CAC7C,MAAuB;IAEvB,OAAO,IAAI,yBAAyB,CAAC,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED,gBAAgB,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC"}
|