edgeflowjs 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -66
- package/dist/backends/index.d.ts +9 -2
- package/dist/backends/index.d.ts.map +1 -1
- package/dist/backends/index.js +13 -13
- package/dist/backends/index.js.map +1 -1
- package/dist/backends/onnx.d.ts +11 -4
- package/dist/backends/onnx.d.ts.map +1 -1
- package/dist/backends/onnx.js +97 -78
- package/dist/backends/onnx.js.map +1 -1
- package/dist/backends/transformers-adapter.d.ts +99 -0
- package/dist/backends/transformers-adapter.d.ts.map +1 -0
- package/dist/backends/transformers-adapter.js +171 -0
- package/dist/backends/transformers-adapter.js.map +1 -0
- package/dist/backends/webgpu.d.ts +7 -5
- package/dist/backends/webgpu.d.ts.map +1 -1
- package/dist/backends/webgpu.js +7 -5
- package/dist/backends/webgpu.js.map +1 -1
- package/dist/backends/webnn.d.ts +6 -5
- package/dist/backends/webnn.d.ts.map +1 -1
- package/dist/backends/webnn.js +6 -5
- package/dist/backends/webnn.js.map +1 -1
- package/dist/core/composer.d.ts +118 -0
- package/dist/core/composer.d.ts.map +1 -0
- package/dist/core/composer.js +163 -0
- package/dist/core/composer.js.map +1 -0
- package/dist/core/device-profiler.d.ts +75 -0
- package/dist/core/device-profiler.d.ts.map +1 -0
- package/dist/core/device-profiler.js +131 -0
- package/dist/core/device-profiler.js.map +1 -0
- package/dist/core/index.d.ts +4 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +8 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/memory.d.ts +22 -2
- package/dist/core/memory.d.ts.map +1 -1
- package/dist/core/memory.js +49 -13
- package/dist/core/memory.js.map +1 -1
- package/dist/core/plugin.d.ts +100 -0
- package/dist/core/plugin.d.ts.map +1 -0
- package/dist/core/plugin.js +106 -0
- package/dist/core/plugin.js.map +1 -0
- package/dist/core/runtime.d.ts +4 -0
- package/dist/core/runtime.d.ts.map +1 -1
- package/dist/core/runtime.js +18 -0
- package/dist/core/runtime.js.map +1 -1
- package/dist/core/scheduler.d.ts +17 -0
- package/dist/core/scheduler.d.ts.map +1 -1
- package/dist/core/scheduler.js +101 -3
- package/dist/core/scheduler.js.map +1 -1
- package/dist/core/types.d.ts +14 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/core/worker.d.ts +202 -0
- package/dist/core/worker.d.ts.map +1 -0
- package/dist/core/worker.js +477 -0
- package/dist/core/worker.js.map +1 -0
- package/dist/edgeflow.browser.js +9770 -4383
- package/dist/edgeflow.browser.js.map +4 -4
- package/dist/edgeflow.browser.min.js +435 -5
- package/dist/edgeflow.browser.min.js.map +4 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +28 -10
- package/dist/index.js.map +1 -1
- package/dist/pipelines/automatic-speech-recognition.d.ts +63 -0
- package/dist/pipelines/automatic-speech-recognition.d.ts.map +1 -0
- package/dist/pipelines/automatic-speech-recognition.js +269 -0
- package/dist/pipelines/automatic-speech-recognition.js.map +1 -0
- package/dist/pipelines/base.d.ts +6 -1
- package/dist/pipelines/base.d.ts.map +1 -1
- package/dist/pipelines/base.js +12 -2
- package/dist/pipelines/base.js.map +1 -1
- package/dist/pipelines/feature-extraction.d.ts +5 -40
- package/dist/pipelines/feature-extraction.d.ts.map +1 -1
- package/dist/pipelines/feature-extraction.js +44 -63
- package/dist/pipelines/feature-extraction.js.map +1 -1
- package/dist/pipelines/image-classification.d.ts +4 -36
- package/dist/pipelines/image-classification.d.ts.map +1 -1
- package/dist/pipelines/image-classification.js +22 -60
- package/dist/pipelines/image-classification.js.map +1 -1
- package/dist/pipelines/image-segmentation.d.ts +221 -0
- package/dist/pipelines/image-segmentation.d.ts.map +1 -0
- package/dist/pipelines/image-segmentation.js +535 -0
- package/dist/pipelines/image-segmentation.js.map +1 -0
- package/dist/pipelines/index.d.ts +18 -0
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +51 -2
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pipelines/object-detection.d.ts +44 -0
- package/dist/pipelines/object-detection.d.ts.map +1 -0
- package/dist/pipelines/object-detection.js +218 -0
- package/dist/pipelines/object-detection.js.map +1 -0
- package/dist/pipelines/question-answering.d.ts +41 -0
- package/dist/pipelines/question-answering.d.ts.map +1 -0
- package/dist/pipelines/question-answering.js +164 -0
- package/dist/pipelines/question-answering.js.map +1 -0
- package/dist/pipelines/text-classification.d.ts +3 -39
- package/dist/pipelines/text-classification.d.ts.map +1 -1
- package/dist/pipelines/text-classification.js +29 -67
- package/dist/pipelines/text-classification.js.map +1 -1
- package/dist/pipelines/text-generation.d.ts +281 -0
- package/dist/pipelines/text-generation.d.ts.map +1 -0
- package/dist/pipelines/text-generation.js +766 -0
- package/dist/pipelines/text-generation.js.map +1 -0
- package/dist/pipelines/zero-shot-classification.d.ts +45 -0
- package/dist/pipelines/zero-shot-classification.d.ts.map +1 -0
- package/dist/pipelines/zero-shot-classification.js +140 -0
- package/dist/pipelines/zero-shot-classification.js.map +1 -0
- package/dist/tools/benchmark.d.ts +92 -0
- package/dist/tools/benchmark.d.ts.map +1 -0
- package/dist/tools/benchmark.js +213 -0
- package/dist/tools/benchmark.js.map +1 -0
- package/dist/tools/debugger.d.ts +258 -0
- package/dist/tools/debugger.d.ts.map +1 -0
- package/dist/tools/debugger.js +624 -0
- package/dist/tools/debugger.js.map +1 -0
- package/dist/tools/index.d.ts +8 -0
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +16 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/monitor.d.ts +284 -0
- package/dist/tools/monitor.d.ts.map +1 -0
- package/dist/tools/monitor.js +921 -0
- package/dist/tools/monitor.js.map +1 -0
- package/dist/tools/quantization.d.ts +235 -0
- package/dist/tools/quantization.d.ts.map +1 -0
- package/dist/tools/quantization.js +830 -0
- package/dist/tools/quantization.js.map +1 -0
- package/dist/utils/hub.d.ts +162 -0
- package/dist/utils/hub.d.ts.map +1 -0
- package/dist/utils/hub.js +311 -0
- package/dist/utils/hub.js.map +1 -0
- package/dist/utils/index.d.ts +3 -1
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +5 -1
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/model-loader.d.ts.map +1 -1
- package/dist/utils/model-loader.js +106 -30
- package/dist/utils/model-loader.js.map +1 -1
- package/dist/utils/offline.d.ts +147 -0
- package/dist/utils/offline.d.ts.map +1 -0
- package/dist/utils/offline.js +405 -0
- package/dist/utils/offline.js.map +1 -0
- package/dist/utils/preprocessor.d.ts +82 -6
- package/dist/utils/preprocessor.d.ts.map +1 -1
- package/dist/utils/preprocessor.js +278 -21
- package/dist/utils/preprocessor.js.map +1 -1
- package/dist/utils/tokenizer.d.ts +197 -72
- package/dist/utils/tokenizer.d.ts.map +1 -1
- package/dist/utils/tokenizer.js +558 -274
- package/dist/utils/tokenizer.js.map +1 -1
- package/package.json +26 -11
|
@@ -1,68 +1,33 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* edgeFlow.js - Feature Extraction Pipeline
|
|
3
3
|
*
|
|
4
|
-
* Extract embeddings/features from text
|
|
4
|
+
* Extract embeddings/features from text using sentence-transformer models.
|
|
5
5
|
*/
|
|
6
6
|
import { PipelineConfig, PipelineOptions } from '../core/types.js';
|
|
7
7
|
import { EdgeFlowTensor } from '../core/tensor.js';
|
|
8
8
|
import { BasePipeline, FeatureExtractionResult } from './base.js';
|
|
9
|
-
/**
|
|
10
|
-
* Feature extraction options
|
|
11
|
-
*/
|
|
12
9
|
export interface FeatureExtractionOptions extends PipelineOptions {
|
|
13
|
-
/** Pooling strategy */
|
|
14
10
|
pooling?: 'mean' | 'max' | 'cls' | 'none';
|
|
15
|
-
/** Normalize embeddings */
|
|
16
11
|
normalize?: boolean;
|
|
17
|
-
/** Output dimension (for dimension reduction) */
|
|
18
12
|
outputDim?: number;
|
|
19
13
|
}
|
|
20
|
-
/**
|
|
21
|
-
* FeatureExtractionPipeline - Extract embeddings from text
|
|
22
|
-
*/
|
|
23
14
|
export declare class FeatureExtractionPipeline extends BasePipeline<string | string[], FeatureExtractionResult | FeatureExtractionResult[]> {
|
|
24
15
|
private tokenizer;
|
|
16
|
+
private onnxModel;
|
|
25
17
|
private embeddingDim;
|
|
18
|
+
private modelUrl;
|
|
19
|
+
private tokenizerUrl;
|
|
20
|
+
private modelInputNames;
|
|
26
21
|
constructor(config: PipelineConfig, embeddingDim?: number);
|
|
27
|
-
/**
|
|
28
|
-
* Initialize pipeline
|
|
29
|
-
*/
|
|
30
22
|
initialize(): Promise<void>;
|
|
31
|
-
/**
|
|
32
|
-
* Run feature extraction
|
|
33
|
-
*/
|
|
34
23
|
run(input: string | string[], options?: FeatureExtractionOptions): Promise<FeatureExtractionResult | FeatureExtractionResult[]>;
|
|
35
|
-
/**
|
|
36
|
-
* Preprocess text input
|
|
37
|
-
*/
|
|
38
24
|
protected preprocess(input: string | string[]): Promise<EdgeFlowTensor[]>;
|
|
39
|
-
/**
|
|
40
|
-
* Run model inference
|
|
41
|
-
*/
|
|
42
25
|
private runInference;
|
|
43
|
-
/**
|
|
44
|
-
* Postprocess model outputs
|
|
45
|
-
*/
|
|
46
26
|
protected postprocess(outputs: EdgeFlowTensor[], options?: FeatureExtractionOptions): Promise<FeatureExtractionResult>;
|
|
47
|
-
/**
|
|
48
|
-
* Extract CLS token embedding
|
|
49
|
-
*/
|
|
50
27
|
private extractCLSEmbedding;
|
|
51
|
-
/**
|
|
52
|
-
* Mean pooling over sequence
|
|
53
|
-
*/
|
|
54
28
|
private meanPooling;
|
|
55
|
-
/**
|
|
56
|
-
* Max pooling over sequence
|
|
57
|
-
*/
|
|
58
29
|
private maxPooling;
|
|
59
|
-
/**
|
|
60
|
-
* L2 normalize vector
|
|
61
|
-
*/
|
|
62
30
|
private normalizeVector;
|
|
63
31
|
}
|
|
64
|
-
/**
|
|
65
|
-
* Create feature extraction pipeline
|
|
66
|
-
*/
|
|
67
32
|
export declare function createFeatureExtractionPipeline(config?: Partial<PipelineConfig>): FeatureExtractionPipeline;
|
|
68
33
|
//# sourceMappingURL=feature-extraction.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"feature-extraction.d.ts","sourceRoot":"","sources":["../../src/pipelines/feature-extraction.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,cAAc,EACd,eAAe,
|
|
1
|
+
{"version":3,"file":"feature-extraction.d.ts","sourceRoot":"","sources":["../../src/pipelines/feature-extraction.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,cAAc,EACd,eAAe,EAEhB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAInD,OAAO,EACL,YAAY,EACZ,uBAAuB,EAExB,MAAM,WAAW,CAAC;AAiBnB,MAAM,WAAW,wBAAyB,SAAQ,eAAe;IAC/D,OAAO,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IAC1C,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,yBAA0B,SAAQ,YAAY,CACzD,MAAM,GAAG,MAAM,EAAE,EACjB,uBAAuB,GAAG,uBAAuB,EAAE,CACpD;IACC,OAAO,CAAC,SAAS,CAA0B;IAC3C,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,eAAe,CAA0B;gBAErC,MAAM,EAAE,cAAc,EAAE,YAAY,GAAE,MAA8B;IAOjE,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B,GAAG,CAChB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,EACxB,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,uBAAuB,GAAG,uBAAuB,EAAE,CAAC;cAwBtC,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;YAsC1E,YAAY;cAYD,WAAW,CAClC,OAAO,EAAE,cAAc,EAAE,EACzB,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,uBAAuB,CAAC;IAsCnC,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,WAAW;IAcnB,OAAO,CAAC,UAAU;IAiBlB,OAAO,CAAC,eAAe;CASxB;AAMD,wBAAgB,+BAA+B,CAC7C,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,GACnC,yBAAyB,CAS3B"}
|
|
@@ -1,33 +1,45 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* edgeFlow.js - Feature Extraction Pipeline
|
|
3
3
|
*
|
|
4
|
-
* Extract embeddings/features from text
|
|
4
|
+
* Extract embeddings/features from text using sentence-transformer models.
|
|
5
5
|
*/
|
|
6
6
|
import { EdgeFlowTensor } from '../core/tensor.js';
|
|
7
|
-
import {
|
|
7
|
+
import { Tokenizer } from '../utils/tokenizer.js';
|
|
8
|
+
import { loadModelData } from '../utils/model-loader.js';
|
|
9
|
+
import { loadModelFromBuffer, runInferenceNamed } from '../core/runtime.js';
|
|
8
10
|
import { BasePipeline, registerPipeline, } from './base.js';
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
// ============================================================================
|
|
12
|
+
// Default Model (all-MiniLM-L6-v2, 384-dim sentence embeddings)
|
|
13
|
+
// ============================================================================
|
|
14
|
+
const DEFAULT_MODELS = {
|
|
15
|
+
model: 'https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/onnx/model_quantized.onnx',
|
|
16
|
+
tokenizer: 'https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/tokenizer.json',
|
|
17
|
+
};
|
|
18
|
+
const DEFAULT_EMBEDDING_DIM = 384;
|
|
12
19
|
export class FeatureExtractionPipeline extends BasePipeline {
|
|
13
20
|
tokenizer = null;
|
|
21
|
+
onnxModel = null;
|
|
14
22
|
embeddingDim;
|
|
15
|
-
|
|
23
|
+
modelUrl;
|
|
24
|
+
tokenizerUrl;
|
|
25
|
+
modelInputNames = new Set();
|
|
26
|
+
constructor(config, embeddingDim = DEFAULT_EMBEDDING_DIM) {
|
|
16
27
|
super(config);
|
|
17
28
|
this.embeddingDim = embeddingDim;
|
|
29
|
+
this.modelUrl = config.model !== 'default' ? config.model : DEFAULT_MODELS.model;
|
|
30
|
+
this.tokenizerUrl = config.tokenizerUrl ?? DEFAULT_MODELS.tokenizer;
|
|
18
31
|
}
|
|
19
|
-
/**
|
|
20
|
-
* Initialize pipeline
|
|
21
|
-
*/
|
|
22
32
|
async initialize() {
|
|
23
33
|
await super.initialize();
|
|
24
34
|
if (!this.tokenizer) {
|
|
25
|
-
this.tokenizer =
|
|
35
|
+
this.tokenizer = await Tokenizer.fromUrl(this.tokenizerUrl);
|
|
36
|
+
}
|
|
37
|
+
if (!this.onnxModel) {
|
|
38
|
+
const modelData = await loadModelData(this.modelUrl, { cache: this.config.cache ?? true });
|
|
39
|
+
this.onnxModel = await loadModelFromBuffer(modelData);
|
|
40
|
+
this.modelInputNames = new Set(this.onnxModel.metadata.inputs.map(i => i.name));
|
|
26
41
|
}
|
|
27
42
|
}
|
|
28
|
-
/**
|
|
29
|
-
* Run feature extraction
|
|
30
|
-
*/
|
|
31
43
|
async run(input, options) {
|
|
32
44
|
const isBatch = Array.isArray(input);
|
|
33
45
|
const inputs = isBatch ? input : [input];
|
|
@@ -35,11 +47,8 @@ export class FeatureExtractionPipeline extends BasePipeline {
|
|
|
35
47
|
const startTime = performance.now();
|
|
36
48
|
const results = [];
|
|
37
49
|
for (const text of inputs) {
|
|
38
|
-
// Preprocess
|
|
39
50
|
const tensorInputs = await this.preprocess(text);
|
|
40
|
-
// Run inference
|
|
41
51
|
const outputs = await this.runInference(tensorInputs);
|
|
42
|
-
// Postprocess
|
|
43
52
|
const result = await this.postprocess(outputs, options);
|
|
44
53
|
results.push(result);
|
|
45
54
|
}
|
|
@@ -49,9 +58,6 @@ export class FeatureExtractionPipeline extends BasePipeline {
|
|
|
49
58
|
}
|
|
50
59
|
return isBatch ? results : results[0];
|
|
51
60
|
}
|
|
52
|
-
/**
|
|
53
|
-
* Preprocess text input
|
|
54
|
-
*/
|
|
55
61
|
async preprocess(input) {
|
|
56
62
|
const text = Array.isArray(input) ? input[0] : input;
|
|
57
63
|
const encoded = this.tokenizer.encode(text, {
|
|
@@ -59,32 +65,28 @@ export class FeatureExtractionPipeline extends BasePipeline {
|
|
|
59
65
|
padding: 'max_length',
|
|
60
66
|
truncation: true,
|
|
61
67
|
});
|
|
62
|
-
const inputIds = new EdgeFlowTensor(
|
|
63
|
-
const attentionMask = new EdgeFlowTensor(
|
|
64
|
-
|
|
68
|
+
const inputIds = new EdgeFlowTensor(BigInt64Array.from(encoded.inputIds.map(id => BigInt(id))), [1, encoded.inputIds.length], 'int64');
|
|
69
|
+
const attentionMask = new EdgeFlowTensor(BigInt64Array.from(encoded.attentionMask.map(m => BigInt(m))), [1, encoded.attentionMask.length], 'int64');
|
|
70
|
+
const tensors = [inputIds, attentionMask];
|
|
71
|
+
// Only emit token_type_ids when the loaded model declares it as an input.
|
|
72
|
+
// XLM-R / RoBERTa / multilingual MiniLM omit this input; feeding it would
|
|
73
|
+
// trip onnxruntime-web's session.run with "invalid input" errors.
|
|
74
|
+
if (this.modelInputNames.has('token_type_ids')) {
|
|
75
|
+
const tokenTypeIds = new EdgeFlowTensor(BigInt64Array.from(encoded.inputIds.map(() => BigInt(0))), [1, encoded.inputIds.length], 'int64');
|
|
76
|
+
tensors.push(tokenTypeIds);
|
|
77
|
+
}
|
|
78
|
+
return tensors;
|
|
65
79
|
}
|
|
66
|
-
/**
|
|
67
|
-
* Run model inference
|
|
68
|
-
*/
|
|
69
80
|
async runInference(inputs) {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const inputData = inputs[0]?.toFloat32Array() ?? new Float32Array(0);
|
|
76
|
-
for (let i = 0; i < seqLen; i++) {
|
|
77
|
-
for (let j = 0; j < this.embeddingDim; j++) {
|
|
78
|
-
const inputVal = inputData[i] ?? 0;
|
|
79
|
-
embeddings[i * this.embeddingDim + j] =
|
|
80
|
-
Math.sin(inputVal * (j + 1) * 0.01) * 0.1;
|
|
81
|
-
}
|
|
81
|
+
const namedInputs = new Map();
|
|
82
|
+
namedInputs.set('input_ids', inputs[0]);
|
|
83
|
+
namedInputs.set('attention_mask', inputs[1]);
|
|
84
|
+
if (this.modelInputNames.has('token_type_ids') && inputs[2]) {
|
|
85
|
+
namedInputs.set('token_type_ids', inputs[2]);
|
|
82
86
|
}
|
|
83
|
-
|
|
87
|
+
const outputs = await runInferenceNamed(this.onnxModel, namedInputs);
|
|
88
|
+
return outputs;
|
|
84
89
|
}
|
|
85
|
-
/**
|
|
86
|
-
* Postprocess model outputs
|
|
87
|
-
*/
|
|
88
90
|
async postprocess(outputs, options) {
|
|
89
91
|
const hiddenStates = outputs[0];
|
|
90
92
|
if (!hiddenStates) {
|
|
@@ -95,44 +97,32 @@ export class FeatureExtractionPipeline extends BasePipeline {
|
|
|
95
97
|
let embeddings;
|
|
96
98
|
switch (pooling) {
|
|
97
99
|
case 'cls':
|
|
98
|
-
// Use first token (CLS) embedding
|
|
99
100
|
embeddings = this.extractCLSEmbedding(hiddenStates);
|
|
100
101
|
break;
|
|
101
102
|
case 'max':
|
|
102
|
-
// Max pooling
|
|
103
103
|
embeddings = this.maxPooling(hiddenStates);
|
|
104
104
|
break;
|
|
105
105
|
case 'none':
|
|
106
|
-
// Return all token embeddings (flattened)
|
|
107
106
|
embeddings = hiddenStates.toArray();
|
|
108
107
|
break;
|
|
109
108
|
case 'mean':
|
|
110
109
|
default:
|
|
111
|
-
// Mean pooling
|
|
112
110
|
embeddings = this.meanPooling(hiddenStates);
|
|
113
111
|
break;
|
|
114
112
|
}
|
|
115
|
-
// Normalize if requested
|
|
116
113
|
if (normalize) {
|
|
117
114
|
embeddings = this.normalizeVector(embeddings);
|
|
118
115
|
}
|
|
119
|
-
// Dimension reduction if requested
|
|
120
116
|
if (options?.outputDim && options.outputDim < embeddings.length) {
|
|
121
117
|
embeddings = embeddings.slice(0, options.outputDim);
|
|
122
118
|
}
|
|
123
119
|
return { embeddings };
|
|
124
120
|
}
|
|
125
|
-
/**
|
|
126
|
-
* Extract CLS token embedding
|
|
127
|
-
*/
|
|
128
121
|
extractCLSEmbedding(hiddenStates) {
|
|
129
122
|
const data = hiddenStates.toFloat32Array();
|
|
130
123
|
const embeddingDim = hiddenStates.shape[2] ?? this.embeddingDim;
|
|
131
124
|
return Array.from(data.slice(0, embeddingDim));
|
|
132
125
|
}
|
|
133
|
-
/**
|
|
134
|
-
* Mean pooling over sequence
|
|
135
|
-
*/
|
|
136
126
|
meanPooling(hiddenStates) {
|
|
137
127
|
const data = hiddenStates.toFloat32Array();
|
|
138
128
|
const seqLen = hiddenStates.shape[1] ?? 1;
|
|
@@ -145,9 +135,6 @@ export class FeatureExtractionPipeline extends BasePipeline {
|
|
|
145
135
|
}
|
|
146
136
|
return Array.from(result);
|
|
147
137
|
}
|
|
148
|
-
/**
|
|
149
|
-
* Max pooling over sequence
|
|
150
|
-
*/
|
|
151
138
|
maxPooling(hiddenStates) {
|
|
152
139
|
const data = hiddenStates.toFloat32Array();
|
|
153
140
|
const seqLen = hiddenStates.shape[1] ?? 1;
|
|
@@ -163,9 +150,6 @@ export class FeatureExtractionPipeline extends BasePipeline {
|
|
|
163
150
|
}
|
|
164
151
|
return result;
|
|
165
152
|
}
|
|
166
|
-
/**
|
|
167
|
-
* L2 normalize vector
|
|
168
|
-
*/
|
|
169
153
|
normalizeVector(vec) {
|
|
170
154
|
let norm = 0;
|
|
171
155
|
for (const v of vec) {
|
|
@@ -180,9 +164,6 @@ export class FeatureExtractionPipeline extends BasePipeline {
|
|
|
180
164
|
// ============================================================================
|
|
181
165
|
// Factory Function
|
|
182
166
|
// ============================================================================
|
|
183
|
-
/**
|
|
184
|
-
* Create feature extraction pipeline
|
|
185
|
-
*/
|
|
186
167
|
export function createFeatureExtractionPipeline(config = {}) {
|
|
187
168
|
return new FeatureExtractionPipeline({
|
|
188
169
|
task: 'feature-extraction',
|
|
@@ -190,8 +171,8 @@ export function createFeatureExtractionPipeline(config = {}) {
|
|
|
190
171
|
runtime: config.runtime,
|
|
191
172
|
cache: config.cache ?? true,
|
|
192
173
|
quantization: config.quantization,
|
|
174
|
+
tokenizerUrl: config.tokenizerUrl,
|
|
193
175
|
});
|
|
194
176
|
}
|
|
195
|
-
// Register pipeline
|
|
196
177
|
registerPipeline('feature-extraction', (config) => new FeatureExtractionPipeline(config));
|
|
197
178
|
//# sourceMappingURL=feature-extraction.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"feature-extraction.js","sourceRoot":"","sources":["../../src/pipelines/feature-extraction.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"feature-extraction.js","sourceRoot":"","sources":["../../src/pipelines/feature-extraction.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAOH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EACL,YAAY,EAEZ,gBAAgB,GACjB,MAAM,WAAW,CAAC;AAEnB,+EAA+E;AAC/E,gEAAgE;AAChE,+EAA+E;AAE/E,MAAM,cAAc,GAAG;IACrB,KAAK,EAAE,uFAAuF;IAC9F,SAAS,EAAE,4EAA4E;CACxF,CAAC;AAEF,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAYlC,MAAM,OAAO,yBAA0B,SAAQ,YAG9C;IACS,SAAS,GAAqB,IAAI,CAAC;IACnC,SAAS,GAAuB,IAAI,CAAC;IACrC,YAAY,CAAS;IACrB,QAAQ,CAAS;IACjB,YAAY,CAAS;IACrB,eAAe,GAAgB,IAAI,GAAG,EAAE,CAAC;IAEjD,YAAY,MAAsB,EAAE,eAAuB,qBAAqB;QAC9E,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;QACjF,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,cAAc,CAAC,SAAS,CAAC;IACtE,CAAC;IAEQ,KAAK,CAAC,UAAU;QACvB,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;YAC3F,IAAI,CAAC,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC;YACtD,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAEQ,KAAK,CAAC,GAAG,CAChB,KAAwB,EACxB,OAAkC;QAElC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAEzC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAExB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,OAAO,GAA8B,EAAE,CAAC;QAE9C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;YACtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,CAAC,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;QAC1D,CAAC;QAED,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;IACzC,CAAC;IAEkB,KAAK,CAAC,UAAU,CAAC,KAAwB;QAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAEtD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,IAAI,EAAE;YAC3C,SAAS,EAAE,GAAG;YACd,OAAO,EAAE,YAAY;YACrB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,cAAc,CACjC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAC1D,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAC5B,OAAO,CACR,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,cAAc,CACtC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAC7D,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EACjC,OAAO,CACR,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAE5D,0EAA0E;QAC1E,0EAA0E;QAC1E,kEAAkE;QAClE,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC/C,MAAM,YAAY,GAAG,IAAI,cAAc,CACrC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EACzD,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAC5B,OAAO,CACR,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,MAAwB;QACjD,MAAM,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;QACtD,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;QACzC,WAAW,CAAC,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;QAC9C,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,WAAW,CAAC,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,SAAU,EAAE,WAAW,CAAC,CAAC;QACtE,OAAO,OAA2B,CAAC;IACrC,CAAC;IAEkB,KAAK,CAAC,WAAW,CAClC,OAAyB,EACzB,OAAkC;QAElC,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC;QAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC;QAE7C,IAAI,UAAoB,CAAC;QAEzB,QAAQ,OAAO,EAAE,CAAC;YAChB,KAAK,KAAK;gBACR,UAAU,GAAG,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;gBACpD,MAAM;YACR,KAAK,KAAK;gBACR,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;gBAC3C,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC;gBACpC,MAAM;YACR,KAAK,MAAM,CAAC;YACZ;gBACE,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;gBAC5C,MAAM;QACV,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,SAAS,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAChE,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,EAAE,UAAU,EAAE,CAAC;IACxB,CAAC;IAEO,mBAAmB,CAAC,YAA4B;QACtD,MAAM,IAAI,GAAG,YAAY,CAAC,cAAc,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC;QAChE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;IACjD,CAAC;IAEO,WAAW,CAAC,YAA4B;QAC9C,MAAM,IAAI,GAAG,YAAY,CAAC,cAAc,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC;QAEhE,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAC;QAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC;YAC5E,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAEO,UAAU,CAAC,YAA4B;QAC7C,MAAM,IAAI,GAAG,YAAY,CAAC,cAAc,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC;QAEhE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAa,CAAC;QACnE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,GAAG,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC5C,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACnC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CAAC,GAAa;QACnC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QAC3B,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAChC,CAAC;CACF;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,MAAM,UAAU,+BAA+B,CAC7C,SAAkC,EAAE;IAEpC,OAAO,IAAI,yBAAyB,CAAC;QACnC,IAAI,EAAE,oBAAoB;QAC1B,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS;QAChC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI;QAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,YAAY,EAAE,MAAM,CAAC,YAAY;KAClC,CAAC,CAAC;AACL,CAAC;AAED,gBAAgB,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC"}
|
|
@@ -6,56 +6,24 @@
|
|
|
6
6
|
import { PipelineConfig, PipelineOptions } from '../core/types.js';
|
|
7
7
|
import { EdgeFlowTensor } from '../core/tensor.js';
|
|
8
8
|
import { BasePipeline, ImageClassificationResult } from './base.js';
|
|
9
|
-
/**
|
|
10
|
-
* Image classification options
|
|
11
|
-
*/
|
|
12
9
|
export interface ImageClassificationOptions extends PipelineOptions {
|
|
13
|
-
/** Return all labels with scores */
|
|
14
10
|
returnAllScores?: boolean;
|
|
15
|
-
/** Custom labels */
|
|
16
11
|
labels?: string[];
|
|
17
|
-
/** Number of top predictions to return */
|
|
18
12
|
topK?: number;
|
|
19
13
|
}
|
|
20
|
-
/**
|
|
21
|
-
* Image classification input types
|
|
22
|
-
*/
|
|
23
14
|
export type ImageInput = HTMLImageElement | HTMLCanvasElement | ImageBitmap | ImageData | string;
|
|
24
|
-
/**
|
|
25
|
-
* ImageClassificationPipeline - Classify images
|
|
26
|
-
*/
|
|
27
15
|
export declare class ImageClassificationPipeline extends BasePipeline<ImageInput | ImageInput[], ImageClassificationResult | ImageClassificationResult[]> {
|
|
28
16
|
private preprocessor;
|
|
17
|
+
private onnxModel;
|
|
29
18
|
private labels;
|
|
30
|
-
private
|
|
31
|
-
constructor(config: PipelineConfig, labels?: string[],
|
|
32
|
-
/**
|
|
33
|
-
* Initialize pipeline
|
|
34
|
-
*/
|
|
19
|
+
private modelUrl;
|
|
20
|
+
constructor(config: PipelineConfig, labels?: string[], _numClasses?: number);
|
|
35
21
|
initialize(): Promise<void>;
|
|
36
|
-
/**
|
|
37
|
-
* Set custom labels
|
|
38
|
-
*/
|
|
39
22
|
setLabels(labels: string[]): void;
|
|
40
|
-
/**
|
|
41
|
-
* Run classification
|
|
42
|
-
*/
|
|
43
23
|
run(input: ImageInput | ImageInput[], options?: ImageClassificationOptions): Promise<ImageClassificationResult | ImageClassificationResult[]>;
|
|
44
|
-
/**
|
|
45
|
-
* Preprocess image input
|
|
46
|
-
*/
|
|
47
24
|
protected preprocess(input: ImageInput | ImageInput[]): Promise<EdgeFlowTensor[]>;
|
|
48
|
-
|
|
49
|
-
* Run model inference
|
|
50
|
-
*/
|
|
51
|
-
private runInference;
|
|
52
|
-
/**
|
|
53
|
-
* Postprocess model outputs
|
|
54
|
-
*/
|
|
25
|
+
private runModelInference;
|
|
55
26
|
protected postprocess(outputs: EdgeFlowTensor[], options?: ImageClassificationOptions): Promise<ImageClassificationResult>;
|
|
56
27
|
}
|
|
57
|
-
/**
|
|
58
|
-
* Create image classification pipeline
|
|
59
|
-
*/
|
|
60
28
|
export declare function createImageClassificationPipeline(config?: Partial<PipelineConfig>, labels?: string[]): ImageClassificationPipeline;
|
|
61
29
|
//# sourceMappingURL=image-classification.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image-classification.d.ts","sourceRoot":"","sources":["../../src/pipelines/image-classification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,cAAc,EACd,eAAe,
|
|
1
|
+
{"version":3,"file":"image-classification.d.ts","sourceRoot":"","sources":["../../src/pipelines/image-classification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,cAAc,EACd,eAAe,EAEhB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAW,MAAM,mBAAmB,CAAC;AAI5D,OAAO,EACL,YAAY,EACZ,yBAAyB,EAG1B,MAAM,WAAW,CAAC;AAenB,MAAM,WAAW,0BAA2B,SAAQ,eAAe;IACjE,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,UAAU,GAClB,gBAAgB,GAChB,iBAAiB,GACjB,WAAW,GACX,SAAS,GACT,MAAM,CAAC;AAEX,qBAAa,2BAA4B,SAAQ,YAAY,CAC3D,UAAU,GAAG,UAAU,EAAE,EACzB,yBAAyB,GAAG,yBAAyB,EAAE,CACxD;IACC,OAAO,CAAC,YAAY,CAAkC;IACtD,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,QAAQ,CAAS;gBAGvB,MAAM,EAAE,cAAc,EACtB,MAAM,CAAC,EAAE,MAAM,EAAE,EACjB,WAAW,GAAE,MAAa;IAOb,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAa1C,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAIlB,GAAG,CAChB,KAAK,EAAE,UAAU,GAAG,UAAU,EAAE,EAChC,OAAO,CAAC,EAAE,0BAA0B,GACnC,OAAO,CAAC,yBAAyB,GAAG,yBAAyB,EAAE,CAAC;cAwB1C,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,EAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;YAUlF,iBAAiB;cAKN,WAAW,CAClC,OAAO,EAAE,cAAc,EAAE,EACzB,OAAO,CAAC,EAAE,0BAA0B,GACnC,OAAO,CAAC,yBAAyB,CAAC;CAuBtC;AAMD,wBAAgB,iCAAiC,CAC/C,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM,EACpC,MAAM,CAAC,EAAE,MAAM,EAAE,GAChB,2BAA2B,CAW7B"}
|
|
@@ -3,40 +3,40 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Classify images into categories using vision models.
|
|
5
5
|
*/
|
|
6
|
-
import {
|
|
6
|
+
import { softmax } from '../core/tensor.js';
|
|
7
7
|
import { createImagePreprocessor } from '../utils/preprocessor.js';
|
|
8
|
+
import { loadModelData } from '../utils/model-loader.js';
|
|
9
|
+
import { loadModelFromBuffer, runInference } from '../core/runtime.js';
|
|
8
10
|
import { BasePipeline, registerPipeline, IMAGENET_LABELS, } from './base.js';
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
// ============================================================================
|
|
12
|
+
// Default Model (MobileViT-small, quantized)
|
|
13
|
+
// ============================================================================
|
|
14
|
+
const DEFAULT_MODELS = {
|
|
15
|
+
model: 'https://huggingface.co/Xenova/mobilevit-small/resolve/main/onnx/model_quantized.onnx',
|
|
16
|
+
};
|
|
12
17
|
export class ImageClassificationPipeline extends BasePipeline {
|
|
13
18
|
preprocessor = null;
|
|
19
|
+
onnxModel = null;
|
|
14
20
|
labels;
|
|
15
|
-
|
|
16
|
-
constructor(config, labels,
|
|
21
|
+
modelUrl;
|
|
22
|
+
constructor(config, labels, _numClasses = 1000) {
|
|
17
23
|
super(config);
|
|
18
24
|
this.labels = labels ?? IMAGENET_LABELS;
|
|
19
|
-
this.
|
|
25
|
+
this.modelUrl = config.model !== 'default' ? config.model : DEFAULT_MODELS.model;
|
|
20
26
|
}
|
|
21
|
-
/**
|
|
22
|
-
* Initialize pipeline
|
|
23
|
-
*/
|
|
24
27
|
async initialize() {
|
|
25
28
|
await super.initialize();
|
|
26
29
|
if (!this.preprocessor) {
|
|
27
30
|
this.preprocessor = createImagePreprocessor('imagenet');
|
|
28
31
|
}
|
|
32
|
+
if (!this.onnxModel) {
|
|
33
|
+
const modelData = await loadModelData(this.modelUrl, { cache: this.config.cache ?? true });
|
|
34
|
+
this.onnxModel = await loadModelFromBuffer(modelData);
|
|
35
|
+
}
|
|
29
36
|
}
|
|
30
|
-
/**
|
|
31
|
-
* Set custom labels
|
|
32
|
-
*/
|
|
33
37
|
setLabels(labels) {
|
|
34
38
|
this.labels = labels;
|
|
35
|
-
this.numClasses = labels.length;
|
|
36
39
|
}
|
|
37
|
-
/**
|
|
38
|
-
* Run classification
|
|
39
|
-
*/
|
|
40
40
|
async run(input, options) {
|
|
41
41
|
const isBatch = Array.isArray(input);
|
|
42
42
|
const inputs = isBatch ? input : [input];
|
|
@@ -44,11 +44,8 @@ export class ImageClassificationPipeline extends BasePipeline {
|
|
|
44
44
|
const startTime = performance.now();
|
|
45
45
|
const results = [];
|
|
46
46
|
for (const image of inputs) {
|
|
47
|
-
// Preprocess
|
|
48
47
|
const tensorInputs = await this.preprocess(image);
|
|
49
|
-
|
|
50
|
-
const outputs = await this.runInference(tensorInputs);
|
|
51
|
-
// Postprocess
|
|
48
|
+
const outputs = await this.runModelInference(tensorInputs);
|
|
52
49
|
const result = await this.postprocess(outputs, options);
|
|
53
50
|
results.push(result);
|
|
54
51
|
}
|
|
@@ -58,53 +55,25 @@ export class ImageClassificationPipeline extends BasePipeline {
|
|
|
58
55
|
}
|
|
59
56
|
return isBatch ? results : results[0];
|
|
60
57
|
}
|
|
61
|
-
/**
|
|
62
|
-
* Preprocess image input
|
|
63
|
-
*/
|
|
64
58
|
async preprocess(input) {
|
|
65
59
|
const image = Array.isArray(input) ? input[0] : input;
|
|
66
|
-
// Process image
|
|
67
60
|
const tensor = await this.preprocessor.process(image);
|
|
68
|
-
// Add batch dimension if needed
|
|
69
61
|
if (tensor.shape.length === 3) {
|
|
70
62
|
return [tensor.reshape([1, ...tensor.shape])];
|
|
71
63
|
}
|
|
72
64
|
return [tensor];
|
|
73
65
|
}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
async runInference(inputs) {
|
|
78
|
-
// Generate mock classification logits for demo
|
|
79
|
-
// In production, this would call the actual model
|
|
80
|
-
const logits = new Float32Array(this.numClasses);
|
|
81
|
-
// Generate deterministic pseudo-logits based on input
|
|
82
|
-
const inputData = inputs[0]?.toFloat32Array() ?? new Float32Array(0);
|
|
83
|
-
let sum = 0;
|
|
84
|
-
for (let i = 0; i < Math.min(1000, inputData.length); i++) {
|
|
85
|
-
sum += inputData[i] ?? 0;
|
|
86
|
-
}
|
|
87
|
-
for (let i = 0; i < this.numClasses; i++) {
|
|
88
|
-
logits[i] = Math.sin(sum * (i + 1) * 0.1) * 3;
|
|
89
|
-
}
|
|
90
|
-
return [new EdgeFlowTensor(logits, [1, this.numClasses], 'float32')];
|
|
66
|
+
async runModelInference(inputs) {
|
|
67
|
+
const outputs = await runInference(this.onnxModel, inputs);
|
|
68
|
+
return outputs;
|
|
91
69
|
}
|
|
92
|
-
/**
|
|
93
|
-
* Postprocess model outputs
|
|
94
|
-
*/
|
|
95
70
|
async postprocess(outputs, options) {
|
|
96
71
|
const logits = outputs[0];
|
|
97
72
|
if (!logits) {
|
|
98
73
|
return { label: 'unknown', score: 0 };
|
|
99
74
|
}
|
|
100
|
-
// Apply softmax
|
|
101
75
|
const probs = softmax(logits, -1);
|
|
102
76
|
const probsArray = probs.toFloat32Array();
|
|
103
|
-
const topK = options?.topK ?? 1;
|
|
104
|
-
if (topK > 1 || options?.returnAllScores) {
|
|
105
|
-
// Return top-K results (simplified to top-1 here)
|
|
106
|
-
}
|
|
107
|
-
// Find argmax
|
|
108
77
|
let maxIdx = 0;
|
|
109
78
|
let maxScore = probsArray[0] ?? 0;
|
|
110
79
|
for (let i = 1; i < probsArray.length; i++) {
|
|
@@ -114,18 +83,12 @@ export class ImageClassificationPipeline extends BasePipeline {
|
|
|
114
83
|
}
|
|
115
84
|
}
|
|
116
85
|
const label = options?.labels?.[maxIdx] ?? this.labels[maxIdx] ?? `class_${maxIdx}`;
|
|
117
|
-
return {
|
|
118
|
-
label,
|
|
119
|
-
score: maxScore,
|
|
120
|
-
};
|
|
86
|
+
return { label, score: maxScore };
|
|
121
87
|
}
|
|
122
88
|
}
|
|
123
89
|
// ============================================================================
|
|
124
90
|
// Factory Function
|
|
125
91
|
// ============================================================================
|
|
126
|
-
/**
|
|
127
|
-
* Create image classification pipeline
|
|
128
|
-
*/
|
|
129
92
|
export function createImageClassificationPipeline(config = {}, labels) {
|
|
130
93
|
return new ImageClassificationPipeline({
|
|
131
94
|
task: 'image-classification',
|
|
@@ -135,6 +98,5 @@ export function createImageClassificationPipeline(config = {}, labels) {
|
|
|
135
98
|
quantization: config.quantization,
|
|
136
99
|
}, labels);
|
|
137
100
|
}
|
|
138
|
-
// Register pipeline
|
|
139
101
|
registerPipeline('image-classification', (config) => new ImageClassificationPipeline(config));
|
|
140
102
|
//# sourceMappingURL=image-classification.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image-classification.js","sourceRoot":"","sources":["../../src/pipelines/image-classification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"image-classification.js","sourceRoot":"","sources":["../../src/pipelines/image-classification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAOH,OAAO,EAAkB,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAqB,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvE,OAAO,EACL,YAAY,EAEZ,gBAAgB,EAChB,eAAe,GAChB,MAAM,WAAW,CAAC;AAEnB,+EAA+E;AAC/E,6CAA6C;AAC7C,+EAA+E;AAE/E,MAAM,cAAc,GAAG;IACrB,KAAK,EAAE,sFAAsF;CAC9F,CAAC;AAoBF,MAAM,OAAO,2BAA4B,SAAQ,YAGhD;IACS,YAAY,GAA6B,IAAI,CAAC;IAC9C,SAAS,GAAuB,IAAI,CAAC;IACrC,MAAM,CAAW;IACjB,QAAQ,CAAS;IAEzB,YACE,MAAsB,EACtB,MAAiB,EACjB,cAAsB,IAAI;QAE1B,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,eAAe,CAAC;QACxC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;IACnF,CAAC;IAEQ,KAAK,CAAC,UAAU;QACvB,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,YAAY,GAAG,uBAAuB,CAAC,UAAU,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC;YAC3F,IAAI,CAAC,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,SAAS,CAAC,MAAgB;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEQ,KAAK,CAAC,GAAG,CAChB,KAAgC,EAChC,OAAoC;QAEpC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAEzC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAExB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,OAAO,GAAgC,EAAE,CAAC;QAEhD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAC3D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,CAAC,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;QAC1D,CAAC;QAED,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;IACzC,CAAC;IAEkB,KAAK,CAAC,UAAU,CAAC,KAAgC;QAClE,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAEvD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,MAAwB;QACtD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,SAAU,EAAE,MAAM,CAAC,CAAC;QAC5D,OAAO,OAA2B,CAAC;IACrC,CAAC;IAEkB,KAAK,CAAC,WAAW,CAClC,OAAyB,EACzB,OAAoC;QAEpC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACxC,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAmB,CAAC;QACpD,MAAM,UAAU,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC;QAE1C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,QAAQ,EAAE,CAAC;gBACpC,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC9B,MAAM,GAAG,CAAC,CAAC;YACb,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,SAAS,MAAM,EAAE,CAAC;QAEpF,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IACpC,CAAC;CACF;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,MAAM,UAAU,iCAAiC,CAC/C,SAAkC,EAAE,EACpC,MAAiB;IAEjB,OAAO,IAAI,2BAA2B,CACpC;QACE,IAAI,EAAE,sBAAsB;QAC5B,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS;QAChC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI;QAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;KAClC,EACD,MAAM,CACP,CAAC;AACJ,CAAC;AAED,gBAAgB,CAAC,sBAAsB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,2BAA2B,CAAC,MAAM,CAAC,CAAC,CAAC"}
|