@pilio/gemini-watermark-remover 1.0.20 → 1.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +67 -2
- package/src/cli/gwrCli.js +1 -1
- package/src/cli/gwrRemoveCommand.js +82 -14
- package/src/core/adaptiveDetector.js +22 -3
- package/src/core/allenkFdncnnDenoise.js +326 -0
- package/src/core/allenkFdncnnNcnnModel.js +244 -0
- package/src/core/allenkFdncnnOnnxExport.js +337 -0
- package/src/core/allenkFdncnnOnnxRuntime.js +138 -0
- package/src/core/allenkFdncnnReferenceRuntime.js +169 -0
- package/src/core/alphaGradientMask.js +165 -0
- package/src/core/blendModes.js +9 -4
- package/src/core/candidateSelector.js +422 -33
- package/src/core/embeddedAlphaMaps.js +2 -0
- package/src/core/geminiSizeCatalog.js +44 -1
- package/src/core/restorationMetrics.js +44 -0
- package/src/core/watermarkEngine.js +2 -1
- package/src/core/watermarkProcessor.js +1048 -82
- package/src/sdk/index.d.ts +29 -1
- package/src/sdk/node.d.ts +13 -0
- package/src/sdk/node.js +7 -0
- package/src/sdk/video.d.ts +84 -0
- package/src/sdk/video.js +263 -0
- package/src/shared/debugFileHandoff.js +109 -0
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
const NCNN_BINARY_PARAM_MAGIC = 7767517;
|
|
2
|
+
const NCNN_LAYER_TYPES = Object.freeze({
|
|
3
|
+
6: 'Convolution',
|
|
4
|
+
16: 'Input'
|
|
5
|
+
});
|
|
6
|
+
const NCNN_WEIGHT_FP16_STORAGE_TAG = 0x01306b47;
|
|
7
|
+
|
|
8
|
+
function readInt32LE(buffer, offset) {
|
|
9
|
+
if (offset + 4 > buffer.length) {
|
|
10
|
+
throw new Error(`Unexpected end of NCNN param at byte ${offset}`);
|
|
11
|
+
}
|
|
12
|
+
return buffer.readInt32LE
|
|
13
|
+
? buffer.readInt32LE(offset)
|
|
14
|
+
: new DataView(buffer.buffer, buffer.byteOffset + offset, 4).getInt32(0, true);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function readUInt32LE(buffer, offset) {
|
|
18
|
+
if (offset + 4 > buffer.length) {
|
|
19
|
+
throw new Error(`Unexpected end of NCNN bin at byte ${offset}`);
|
|
20
|
+
}
|
|
21
|
+
return buffer.readUInt32LE
|
|
22
|
+
? buffer.readUInt32LE(offset)
|
|
23
|
+
: new DataView(buffer.buffer, buffer.byteOffset + offset, 4).getUint32(0, true);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function normalizeBuffer(buffer) {
|
|
27
|
+
if (buffer instanceof Uint8Array) return buffer;
|
|
28
|
+
return new Uint8Array(buffer);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function parseParamPairs(buffer, cursor) {
|
|
32
|
+
const params = {};
|
|
33
|
+
let offset = cursor;
|
|
34
|
+
|
|
35
|
+
while (offset < buffer.length) {
|
|
36
|
+
const key = readInt32LE(buffer, offset);
|
|
37
|
+
offset += 4;
|
|
38
|
+
if (key === -233) {
|
|
39
|
+
return { params, offset };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const value = readInt32LE(buffer, offset);
|
|
43
|
+
offset += 4;
|
|
44
|
+
params[key] = value;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
throw new Error('NCNN layer params were not terminated by -233');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function getNcnnLayerTypeName(typeIndex) {
|
|
51
|
+
return NCNN_LAYER_TYPES[typeIndex] || `LayerType${typeIndex}`;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function normalizeConvolutionParams(params = {}) {
|
|
55
|
+
return {
|
|
56
|
+
numOutput: params[0] ?? 0,
|
|
57
|
+
kernelW: params[1] ?? 0,
|
|
58
|
+
dilationW: params[2] ?? 1,
|
|
59
|
+
strideW: params[3] ?? 1,
|
|
60
|
+
padW: params[4] ?? 0,
|
|
61
|
+
biasTerm: params[5] ?? 0,
|
|
62
|
+
weightDataSize: params[6] ?? 0,
|
|
63
|
+
activationType: params[9] ?? 0,
|
|
64
|
+
kernelH: params[11] ?? params[1] ?? 0,
|
|
65
|
+
dilationH: params[12] ?? params[2] ?? 1,
|
|
66
|
+
strideH: params[13] ?? params[3] ?? 1,
|
|
67
|
+
padH: params[14] ?? params[4] ?? 0
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function parseAllenkFdncnnParam(bufferLike) {
|
|
72
|
+
const buffer = normalizeBuffer(bufferLike);
|
|
73
|
+
let offset = 0;
|
|
74
|
+
|
|
75
|
+
const magic = readInt32LE(buffer, offset);
|
|
76
|
+
offset += 4;
|
|
77
|
+
if (magic !== NCNN_BINARY_PARAM_MAGIC) {
|
|
78
|
+
throw new Error(`Unsupported NCNN binary param magic: ${magic}`);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const layerCount = readInt32LE(buffer, offset);
|
|
82
|
+
offset += 4;
|
|
83
|
+
const blobCount = readInt32LE(buffer, offset);
|
|
84
|
+
offset += 4;
|
|
85
|
+
const layers = [];
|
|
86
|
+
|
|
87
|
+
for (let i = 0; i < layerCount; i++) {
|
|
88
|
+
const typeIndex = readInt32LE(buffer, offset);
|
|
89
|
+
offset += 4;
|
|
90
|
+
const bottomCount = readInt32LE(buffer, offset);
|
|
91
|
+
offset += 4;
|
|
92
|
+
const topCount = readInt32LE(buffer, offset);
|
|
93
|
+
offset += 4;
|
|
94
|
+
const bottoms = [];
|
|
95
|
+
const tops = [];
|
|
96
|
+
|
|
97
|
+
for (let b = 0; b < bottomCount; b++) {
|
|
98
|
+
bottoms.push(readInt32LE(buffer, offset));
|
|
99
|
+
offset += 4;
|
|
100
|
+
}
|
|
101
|
+
for (let t = 0; t < topCount; t++) {
|
|
102
|
+
tops.push(readInt32LE(buffer, offset));
|
|
103
|
+
offset += 4;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const parsed = parseParamPairs(buffer, offset);
|
|
107
|
+
offset = parsed.offset;
|
|
108
|
+
|
|
109
|
+
const type = getNcnnLayerTypeName(typeIndex);
|
|
110
|
+
const layer = {
|
|
111
|
+
index: i,
|
|
112
|
+
typeIndex,
|
|
113
|
+
type,
|
|
114
|
+
bottoms,
|
|
115
|
+
tops,
|
|
116
|
+
params: parsed.params
|
|
117
|
+
};
|
|
118
|
+
if (type === 'Convolution') {
|
|
119
|
+
layer.convolution = normalizeConvolutionParams(parsed.params);
|
|
120
|
+
}
|
|
121
|
+
layers.push(layer);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
return {
|
|
125
|
+
magic,
|
|
126
|
+
layerCount,
|
|
127
|
+
blobCount,
|
|
128
|
+
layers,
|
|
129
|
+
bytesRead: offset,
|
|
130
|
+
byteLength: buffer.length
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function halfToFloat(half) {
|
|
135
|
+
const sign = (half & 0x8000) ? -1 : 1;
|
|
136
|
+
const exponent = (half >> 10) & 0x1f;
|
|
137
|
+
const fraction = half & 0x03ff;
|
|
138
|
+
|
|
139
|
+
if (exponent === 0) {
|
|
140
|
+
return sign * Math.pow(2, -14) * (fraction / 1024);
|
|
141
|
+
}
|
|
142
|
+
if (exponent === 0x1f) {
|
|
143
|
+
return fraction ? Number.NaN : sign * Number.POSITIVE_INFINITY;
|
|
144
|
+
}
|
|
145
|
+
return sign * Math.pow(2, exponent - 15) * (1 + fraction / 1024);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function buildAllenkFdncnnWeightLayout(param, binLike) {
|
|
149
|
+
const bin = normalizeBuffer(binLike);
|
|
150
|
+
let offset = 0;
|
|
151
|
+
let inputChannels = 4;
|
|
152
|
+
const segments = [];
|
|
153
|
+
|
|
154
|
+
for (const layer of param.layers) {
|
|
155
|
+
if (layer.type !== 'Convolution') {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const conv = layer.convolution;
|
|
160
|
+
const expectedWeightCount = conv.numOutput * inputChannels * conv.kernelW * conv.kernelH;
|
|
161
|
+
if (expectedWeightCount !== conv.weightDataSize) {
|
|
162
|
+
throw new Error(
|
|
163
|
+
`Unexpected weight count for layer ${layer.index}: expected ${expectedWeightCount}, got ${conv.weightDataSize}`
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
const storageTag = readUInt32LE(bin, offset);
|
|
168
|
+
if (storageTag !== NCNN_WEIGHT_FP16_STORAGE_TAG) {
|
|
169
|
+
throw new Error(`Unexpected NCNN weight storage tag at layer ${layer.index}: 0x${storageTag.toString(16)}`);
|
|
170
|
+
}
|
|
171
|
+
const weightOffset = offset + 4;
|
|
172
|
+
const weightBytes = conv.weightDataSize * 2;
|
|
173
|
+
const biasOffset = weightOffset + weightBytes;
|
|
174
|
+
const biasCount = conv.biasTerm ? conv.numOutput : 0;
|
|
175
|
+
const biasBytes = biasCount * 4;
|
|
176
|
+
|
|
177
|
+
if (biasOffset + biasBytes > bin.length) {
|
|
178
|
+
throw new Error(`NCNN weights exceed bin length at layer ${layer.index}`);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
segments.push({
|
|
182
|
+
layerIndex: layer.index,
|
|
183
|
+
type: layer.type,
|
|
184
|
+
inputChannels,
|
|
185
|
+
outputChannels: conv.numOutput,
|
|
186
|
+
kernelW: conv.kernelW,
|
|
187
|
+
kernelH: conv.kernelH,
|
|
188
|
+
strideW: conv.strideW,
|
|
189
|
+
strideH: conv.strideH,
|
|
190
|
+
padW: conv.padW,
|
|
191
|
+
padH: conv.padH,
|
|
192
|
+
activationType: conv.activationType,
|
|
193
|
+
storageTag,
|
|
194
|
+
weightOffset,
|
|
195
|
+
weightBytes,
|
|
196
|
+
weightCount: conv.weightDataSize,
|
|
197
|
+
biasOffset,
|
|
198
|
+
biasBytes,
|
|
199
|
+
biasCount
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
offset = biasOffset + biasBytes;
|
|
203
|
+
inputChannels = conv.numOutput;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
return {
|
|
207
|
+
storage: 'fp16-weights-fp32-bias',
|
|
208
|
+
byteLength: bin.length,
|
|
209
|
+
bytesRead: offset,
|
|
210
|
+
segments
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function summarizeAllenkFdncnnModel(param, weightLayout = null) {
|
|
215
|
+
const convolutionLayers = param.layers.filter((layer) => layer.type === 'Convolution');
|
|
216
|
+
const reluLayers = convolutionLayers.filter((layer) => layer.convolution?.activationType === 1);
|
|
217
|
+
const outputLayer = convolutionLayers[convolutionLayers.length - 1] || null;
|
|
218
|
+
|
|
219
|
+
return {
|
|
220
|
+
layerCount: param.layerCount,
|
|
221
|
+
blobCount: param.blobCount,
|
|
222
|
+
convolutionLayerCount: convolutionLayers.length,
|
|
223
|
+
reluConvolutionLayerCount: reluLayers.length,
|
|
224
|
+
inputBlob: 0,
|
|
225
|
+
outputBlob: outputLayer?.tops?.[0] ?? null,
|
|
226
|
+
inputChannels: weightLayout?.segments?.[0]?.inputChannels ?? 4,
|
|
227
|
+
hiddenChannels: convolutionLayers[0]?.convolution?.numOutput ?? null,
|
|
228
|
+
outputChannels: outputLayer?.convolution?.numOutput ?? null,
|
|
229
|
+
kernel: convolutionLayers[0]
|
|
230
|
+
? `${convolutionLayers[0].convolution.kernelW}x${convolutionLayers[0].convolution.kernelH}`
|
|
231
|
+
: null,
|
|
232
|
+
bytesRead: weightLayout?.bytesRead ?? null,
|
|
233
|
+
weightBinBytes: weightLayout?.byteLength ?? null
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export {
|
|
238
|
+
NCNN_BINARY_PARAM_MAGIC,
|
|
239
|
+
NCNN_WEIGHT_FP16_STORAGE_TAG,
|
|
240
|
+
buildAllenkFdncnnWeightLayout,
|
|
241
|
+
halfToFloat,
|
|
242
|
+
parseAllenkFdncnnParam,
|
|
243
|
+
summarizeAllenkFdncnnModel
|
|
244
|
+
};
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
import { halfToFloat } from './allenkFdncnnNcnnModel.js';
|
|
2
|
+
|
|
3
|
+
const ONNX_IR_VERSION = 8;
|
|
4
|
+
const ONNX_OPSET_VERSION = 13;
|
|
5
|
+
const ONNX_TENSOR_FLOAT = 1;
|
|
6
|
+
const ONNX_ATTR_INTS = 7;
|
|
7
|
+
|
|
8
|
+
const WIRE_VARINT = 0;
|
|
9
|
+
const WIRE_LENGTH_DELIMITED = 2;
|
|
10
|
+
|
|
11
|
+
function normalizeBytes(bytes) {
|
|
12
|
+
if (bytes instanceof Uint8Array) return bytes;
|
|
13
|
+
return new Uint8Array(bytes);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function concatBytes(parts) {
|
|
17
|
+
const total = parts.reduce((sum, part) => sum + part.length, 0);
|
|
18
|
+
const out = new Uint8Array(total);
|
|
19
|
+
let offset = 0;
|
|
20
|
+
for (const part of parts) {
|
|
21
|
+
out.set(part, offset);
|
|
22
|
+
offset += part.length;
|
|
23
|
+
}
|
|
24
|
+
return out;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function encodeVarint(value) {
|
|
28
|
+
let next = BigInt(value);
|
|
29
|
+
if (next < 0n) {
|
|
30
|
+
throw new Error(`Cannot encode negative varint: ${value}`);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const bytes = [];
|
|
34
|
+
while (next >= 0x80n) {
|
|
35
|
+
bytes.push(Number((next & 0x7fn) | 0x80n));
|
|
36
|
+
next >>= 7n;
|
|
37
|
+
}
|
|
38
|
+
bytes.push(Number(next));
|
|
39
|
+
return Uint8Array.from(bytes);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function fieldKey(fieldNumber, wireType) {
|
|
43
|
+
return encodeVarint((BigInt(fieldNumber) << 3n) | BigInt(wireType));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function varintField(fieldNumber, value) {
|
|
47
|
+
return concatBytes([fieldKey(fieldNumber, WIRE_VARINT), encodeVarint(value)]);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function int64Field(fieldNumber, value) {
|
|
51
|
+
return varintField(fieldNumber, BigInt(value));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function bytesField(fieldNumber, bytesLike) {
|
|
55
|
+
const bytes = normalizeBytes(bytesLike);
|
|
56
|
+
return concatBytes([
|
|
57
|
+
fieldKey(fieldNumber, WIRE_LENGTH_DELIMITED),
|
|
58
|
+
encodeVarint(bytes.length),
|
|
59
|
+
bytes
|
|
60
|
+
]);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function stringField(fieldNumber, value) {
|
|
64
|
+
return bytesField(fieldNumber, new TextEncoder().encode(String(value)));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function messageField(fieldNumber, message) {
|
|
68
|
+
return bytesField(fieldNumber, message);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function message(parts) {
|
|
72
|
+
return concatBytes(parts.filter(Boolean));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function float32RawData(values) {
|
|
76
|
+
const out = new Uint8Array(values.length * 4);
|
|
77
|
+
const view = new DataView(out.buffer);
|
|
78
|
+
for (let i = 0; i < values.length; i++) {
|
|
79
|
+
view.setFloat32(i * 4, values[i], true);
|
|
80
|
+
}
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function readFloat32LE(bytes, offset) {
|
|
85
|
+
const buffer = normalizeBytes(bytes);
|
|
86
|
+
return new DataView(buffer.buffer, buffer.byteOffset + offset, 4).getFloat32(0, true);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function readUInt16LE(bytes, offset) {
|
|
90
|
+
const buffer = normalizeBytes(bytes);
|
|
91
|
+
return new DataView(buffer.buffer, buffer.byteOffset + offset, 2).getUint16(0, true);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function createTensorProto({ name, dims, rawData }) {
|
|
95
|
+
return message([
|
|
96
|
+
...dims.map((dim) => int64Field(1, dim)),
|
|
97
|
+
varintField(2, ONNX_TENSOR_FLOAT),
|
|
98
|
+
stringField(8, name),
|
|
99
|
+
bytesField(9, rawData)
|
|
100
|
+
]);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function createShapeProto(dims) {
|
|
104
|
+
return message(dims.map((dim) => messageField(1, message([int64Field(1, dim)]))));
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function createValueInfoProto({ name, dims }) {
|
|
108
|
+
const tensorType = message([
|
|
109
|
+
varintField(1, ONNX_TENSOR_FLOAT),
|
|
110
|
+
messageField(2, createShapeProto(dims))
|
|
111
|
+
]);
|
|
112
|
+
const typeProto = message([messageField(1, tensorType)]);
|
|
113
|
+
|
|
114
|
+
return message([
|
|
115
|
+
stringField(1, name),
|
|
116
|
+
messageField(2, typeProto)
|
|
117
|
+
]);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function createIntsAttribute(name, values) {
|
|
121
|
+
return message([
|
|
122
|
+
stringField(1, name),
|
|
123
|
+
...values.map((value) => int64Field(8, value)),
|
|
124
|
+
varintField(20, ONNX_ATTR_INTS)
|
|
125
|
+
]);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function createNodeProto({ name, opType, inputs, outputs, attributes = [] }) {
|
|
129
|
+
return message([
|
|
130
|
+
...inputs.map((input) => stringField(1, input)),
|
|
131
|
+
...outputs.map((output) => stringField(2, output)),
|
|
132
|
+
stringField(3, name),
|
|
133
|
+
stringField(4, opType),
|
|
134
|
+
...attributes.map((attribute) => messageField(5, attribute))
|
|
135
|
+
]);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function decodeSegmentWeightsToFloat32(bin, segment) {
|
|
139
|
+
const values = new Array(segment.weightCount);
|
|
140
|
+
for (let i = 0; i < segment.weightCount; i++) {
|
|
141
|
+
values[i] = halfToFloat(readUInt16LE(bin, segment.weightOffset + i * 2));
|
|
142
|
+
}
|
|
143
|
+
return float32RawData(values);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function decodeSegmentBiasToFloat32(bin, segment) {
|
|
147
|
+
const values = new Array(segment.biasCount);
|
|
148
|
+
for (let i = 0; i < segment.biasCount; i++) {
|
|
149
|
+
values[i] = readFloat32LE(bin, segment.biasOffset + i * 4);
|
|
150
|
+
}
|
|
151
|
+
return float32RawData(values);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function createInitializers({ bin, segments }) {
|
|
155
|
+
const initializers = [];
|
|
156
|
+
|
|
157
|
+
for (let i = 0; i < segments.length; i++) {
|
|
158
|
+
const segment = segments[i];
|
|
159
|
+
initializers.push({
|
|
160
|
+
name: `conv${i + 1}.weight`,
|
|
161
|
+
tensor: createTensorProto({
|
|
162
|
+
name: `conv${i + 1}.weight`,
|
|
163
|
+
dims: [
|
|
164
|
+
segment.outputChannels,
|
|
165
|
+
segment.inputChannels,
|
|
166
|
+
segment.kernelH,
|
|
167
|
+
segment.kernelW
|
|
168
|
+
],
|
|
169
|
+
rawData: decodeSegmentWeightsToFloat32(bin, segment)
|
|
170
|
+
})
|
|
171
|
+
});
|
|
172
|
+
initializers.push({
|
|
173
|
+
name: `conv${i + 1}.bias`,
|
|
174
|
+
tensor: createTensorProto({
|
|
175
|
+
name: `conv${i + 1}.bias`,
|
|
176
|
+
dims: [segment.biasCount],
|
|
177
|
+
rawData: decodeSegmentBiasToFloat32(bin, segment)
|
|
178
|
+
})
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return initializers;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function createNodes({ segments, inputName, outputName }) {
|
|
186
|
+
const nodes = [];
|
|
187
|
+
let previous = inputName;
|
|
188
|
+
|
|
189
|
+
for (let i = 0; i < segments.length; i++) {
|
|
190
|
+
const segment = segments[i];
|
|
191
|
+
const index = i + 1;
|
|
192
|
+
const isLast = i === segments.length - 1;
|
|
193
|
+
const convOutput = isLast ? outputName : `conv${index}.out`;
|
|
194
|
+
const reluOutput = `relu${index}.out`;
|
|
195
|
+
|
|
196
|
+
nodes.push(createNodeProto({
|
|
197
|
+
name: `conv${index}`,
|
|
198
|
+
opType: 'Conv',
|
|
199
|
+
inputs: [
|
|
200
|
+
previous,
|
|
201
|
+
`conv${index}.weight`,
|
|
202
|
+
`conv${index}.bias`
|
|
203
|
+
],
|
|
204
|
+
outputs: [convOutput],
|
|
205
|
+
attributes: [
|
|
206
|
+
createIntsAttribute('kernel_shape', [segment.kernelH, segment.kernelW]),
|
|
207
|
+
createIntsAttribute('pads', [segment.padH, segment.padW, segment.padH, segment.padW]),
|
|
208
|
+
createIntsAttribute('strides', [segment.strideH, segment.strideW])
|
|
209
|
+
]
|
|
210
|
+
}));
|
|
211
|
+
|
|
212
|
+
if (!isLast && segment.activationType === 1) {
|
|
213
|
+
nodes.push(createNodeProto({
|
|
214
|
+
name: `relu${index}`,
|
|
215
|
+
opType: 'Relu',
|
|
216
|
+
inputs: [convOutput],
|
|
217
|
+
outputs: [reluOutput]
|
|
218
|
+
}));
|
|
219
|
+
previous = reluOutput;
|
|
220
|
+
} else {
|
|
221
|
+
previous = convOutput;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
return nodes;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function createGraphProto({
|
|
229
|
+
bin,
|
|
230
|
+
name,
|
|
231
|
+
segments,
|
|
232
|
+
inputName,
|
|
233
|
+
outputName,
|
|
234
|
+
roiSize
|
|
235
|
+
}) {
|
|
236
|
+
const initializers = createInitializers({ bin, segments });
|
|
237
|
+
const nodes = createNodes({ segments, inputName, outputName });
|
|
238
|
+
const inputShape = [1, segments[0].inputChannels, roiSize, roiSize];
|
|
239
|
+
const outputShape = [1, segments[segments.length - 1].outputChannels, roiSize, roiSize];
|
|
240
|
+
|
|
241
|
+
return {
|
|
242
|
+
nodeCount: nodes.length,
|
|
243
|
+
initializerCount: initializers.length,
|
|
244
|
+
bytes: message([
|
|
245
|
+
...nodes.map((node) => messageField(1, node)),
|
|
246
|
+
stringField(2, name),
|
|
247
|
+
...initializers.map((initializer) => messageField(5, initializer.tensor)),
|
|
248
|
+
messageField(11, createValueInfoProto({ name: inputName, dims: inputShape })),
|
|
249
|
+
messageField(12, createValueInfoProto({ name: outputName, dims: outputShape }))
|
|
250
|
+
])
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function createOpsetImport(version = ONNX_OPSET_VERSION) {
|
|
255
|
+
return message([int64Field(2, version)]);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function createModelProto({
|
|
259
|
+
graph,
|
|
260
|
+
irVersion = ONNX_IR_VERSION,
|
|
261
|
+
opsetVersion = ONNX_OPSET_VERSION,
|
|
262
|
+
producerName = 'gemini-watermark-remover',
|
|
263
|
+
modelVersion = 1
|
|
264
|
+
}) {
|
|
265
|
+
return message([
|
|
266
|
+
int64Field(1, irVersion),
|
|
267
|
+
stringField(2, producerName),
|
|
268
|
+
int64Field(5, modelVersion),
|
|
269
|
+
messageField(7, graph),
|
|
270
|
+
messageField(8, createOpsetImport(opsetVersion))
|
|
271
|
+
]);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function exportAllenkFdncnnOnnx({
|
|
275
|
+
bin,
|
|
276
|
+
weightLayout,
|
|
277
|
+
roiSize = 72,
|
|
278
|
+
inputName = 'fdncnn_input',
|
|
279
|
+
outputName = 'fdncnn_output',
|
|
280
|
+
graphName = 'allenk_fdncnn_color',
|
|
281
|
+
opsetVersion = ONNX_OPSET_VERSION,
|
|
282
|
+
irVersion = ONNX_IR_VERSION
|
|
283
|
+
} = {}) {
|
|
284
|
+
const segments = weightLayout?.segments || [];
|
|
285
|
+
if (!segments.length) {
|
|
286
|
+
throw new Error('allenk FDnCNN ONNX export requires weightLayout.segments');
|
|
287
|
+
}
|
|
288
|
+
if (!Number.isInteger(roiSize) || roiSize <= 0) {
|
|
289
|
+
throw new Error(`Invalid ONNX ROI size: ${roiSize}`);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
const graph = createGraphProto({
|
|
293
|
+
bin: normalizeBytes(bin),
|
|
294
|
+
name: graphName,
|
|
295
|
+
segments,
|
|
296
|
+
inputName,
|
|
297
|
+
outputName,
|
|
298
|
+
roiSize
|
|
299
|
+
});
|
|
300
|
+
const model = createModelProto({
|
|
301
|
+
graph: graph.bytes,
|
|
302
|
+
irVersion,
|
|
303
|
+
opsetVersion
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
return {
|
|
307
|
+
bytes: model,
|
|
308
|
+
metadata: {
|
|
309
|
+
format: 'onnx',
|
|
310
|
+
tensorDataType: 'float32-raw-data',
|
|
311
|
+
irVersion,
|
|
312
|
+
opsetVersion,
|
|
313
|
+
graphName,
|
|
314
|
+
inputName,
|
|
315
|
+
outputName,
|
|
316
|
+
inputShape: [1, segments[0].inputChannels, roiSize, roiSize],
|
|
317
|
+
outputShape: [1, segments[segments.length - 1].outputChannels, roiSize, roiSize],
|
|
318
|
+
nodeCount: graph.nodeCount,
|
|
319
|
+
convolutionNodeCount: segments.length,
|
|
320
|
+
reluNodeCount: segments.filter((segment, index) => index < segments.length - 1 && segment.activationType === 1).length,
|
|
321
|
+
initializerCount: graph.initializerCount,
|
|
322
|
+
roiSize
|
|
323
|
+
}
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
export {
|
|
328
|
+
ONNX_IR_VERSION,
|
|
329
|
+
ONNX_OPSET_VERSION,
|
|
330
|
+
concatBytes,
|
|
331
|
+
createIntsAttribute,
|
|
332
|
+
createNodeProto,
|
|
333
|
+
createTensorProto,
|
|
334
|
+
encodeVarint,
|
|
335
|
+
exportAllenkFdncnnOnnx,
|
|
336
|
+
varintField
|
|
337
|
+
};
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import * as wasmOrt from 'onnxruntime-web/wasm';
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
buildAllenkFdncnnInput,
|
|
5
|
+
convertAllenkFdncnnOutputToRgba
|
|
6
|
+
} from './allenkFdncnnDenoise.js';
|
|
7
|
+
|
|
8
|
+
function getNow() {
|
|
9
|
+
return globalThis.performance?.now ? globalThis.performance.now() : Date.now();
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function normalizeShape(shape, fallback = null) {
|
|
13
|
+
if (!Array.isArray(shape)) return fallback;
|
|
14
|
+
const normalized = shape.map((value) => Number(value));
|
|
15
|
+
return normalized.every((value) => Number.isInteger(value) && value > 0)
|
|
16
|
+
? normalized
|
|
17
|
+
: fallback;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function validateImageShape(imageData, inputShape) {
|
|
21
|
+
const expectedHeight = inputShape?.[2];
|
|
22
|
+
const expectedWidth = inputShape?.[3];
|
|
23
|
+
if (!imageData?.data || imageData.width <= 0 || imageData.height <= 0) {
|
|
24
|
+
throw new Error('allenk FDnCNN ONNX runtime requires ImageData-like input');
|
|
25
|
+
}
|
|
26
|
+
if (imageData.width !== expectedWidth || imageData.height !== expectedHeight) {
|
|
27
|
+
const error = new Error(
|
|
28
|
+
`allenk FDnCNN ONNX runtime expected ${expectedWidth}x${expectedHeight}, got ${imageData.width}x${imageData.height}`
|
|
29
|
+
);
|
|
30
|
+
error.code = 'ALLENK_FDNCNN_ONNX_SHAPE_MISMATCH';
|
|
31
|
+
error.expectedWidth = expectedWidth;
|
|
32
|
+
error.expectedHeight = expectedHeight;
|
|
33
|
+
error.actualWidth = imageData.width;
|
|
34
|
+
error.actualHeight = imageData.height;
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function createAllenkFdncnnOnnxRuntime({
|
|
40
|
+
ort = null,
|
|
41
|
+
modelBytes,
|
|
42
|
+
session = null,
|
|
43
|
+
executionProvider = 'wasm',
|
|
44
|
+
inputName = 'fdncnn_input',
|
|
45
|
+
outputName = 'fdncnn_output',
|
|
46
|
+
inputShape = [1, 4, 72, 72],
|
|
47
|
+
outputShape = [1, 3, 72, 72],
|
|
48
|
+
graphOptimizationLevel = 'all',
|
|
49
|
+
wasmPaths = null,
|
|
50
|
+
numThreads = 'auto'
|
|
51
|
+
} = {}) {
|
|
52
|
+
if (executionProvider === 'webgpu' && !ort) {
|
|
53
|
+
throw new Error('allenk FDnCNN WebGPU runtime requires an injected onnxruntime-web/webgpu module');
|
|
54
|
+
}
|
|
55
|
+
const resolvedOrt = ort || wasmOrt;
|
|
56
|
+
const resolvedInputShape = normalizeShape(inputShape, [1, 4, 72, 72]);
|
|
57
|
+
const resolvedOutputShape = normalizeShape(outputShape, [1, 3, resolvedInputShape[2], resolvedInputShape[3]]);
|
|
58
|
+
if (resolvedOrt.env?.wasm) {
|
|
59
|
+
const canUseThreads = Boolean(globalThis.crossOriginIsolated && typeof SharedArrayBuffer !== 'undefined');
|
|
60
|
+
const hardwareThreads = Number(globalThis.navigator?.hardwareConcurrency) || 1;
|
|
61
|
+
const resolvedNumThreads = numThreads === 'auto'
|
|
62
|
+
? (canUseThreads ? Math.max(1, Math.min(4, Math.ceil(hardwareThreads / 2))) : 1)
|
|
63
|
+
: Math.max(1, Math.round(Number(numThreads) || 1));
|
|
64
|
+
resolvedOrt.env.wasm.numThreads = executionProvider === 'wasm' ? resolvedNumThreads : 1;
|
|
65
|
+
resolvedOrt.env.wasm.proxy = false;
|
|
66
|
+
if (wasmPaths) {
|
|
67
|
+
resolvedOrt.env.wasm.wasmPaths = wasmPaths;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (executionProvider === 'webgpu' && resolvedOrt.env?.webgpu) {
|
|
71
|
+
resolvedOrt.env.webgpu.powerPreference = 'high-performance';
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const createStarted = getNow();
|
|
75
|
+
const resolvedSession = session || await resolvedOrt.InferenceSession.create(modelBytes, {
|
|
76
|
+
executionProviders: [executionProvider],
|
|
77
|
+
graphOptimizationLevel
|
|
78
|
+
});
|
|
79
|
+
const createMs = getNow() - createStarted;
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
id: `allenk-fdncnn-onnx-${executionProvider}`,
|
|
83
|
+
status: 'prototype',
|
|
84
|
+
executionProvider,
|
|
85
|
+
inputName,
|
|
86
|
+
outputName,
|
|
87
|
+
inputShape: resolvedInputShape,
|
|
88
|
+
outputShape: resolvedOutputShape,
|
|
89
|
+
createMs,
|
|
90
|
+
session: resolvedSession,
|
|
91
|
+
numThreads: executionProvider === 'wasm'
|
|
92
|
+
? resolvedOrt.env?.wasm?.numThreads ?? null
|
|
93
|
+
: null,
|
|
94
|
+
estimateMacs(width, height) {
|
|
95
|
+
// Matches the decoded allenk FDnCNN graph: 4->64, 18x 64->64, 64->3, all 3x3.
|
|
96
|
+
return width * height * ((4 * 64 * 9) + (18 * 64 * 64 * 9) + (64 * 3 * 9));
|
|
97
|
+
},
|
|
98
|
+
async execute(input) {
|
|
99
|
+
const tensor = new resolvedOrt.Tensor('float32', input, resolvedInputShape);
|
|
100
|
+
const started = getNow();
|
|
101
|
+
const outputs = await resolvedSession.run({ [inputName]: tensor });
|
|
102
|
+
const runMs = getNow() - started;
|
|
103
|
+
const outputTensor = outputs[outputName];
|
|
104
|
+
if (!outputTensor?.data) {
|
|
105
|
+
throw new Error(`allenk FDnCNN ONNX runtime did not return ${outputName}`);
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
output: outputTensor.data,
|
|
109
|
+
outputShape: [...outputTensor.dims],
|
|
110
|
+
runtime: this.id,
|
|
111
|
+
runMs,
|
|
112
|
+
macs: this.estimateMacs(resolvedInputShape[3], resolvedInputShape[2])
|
|
113
|
+
};
|
|
114
|
+
},
|
|
115
|
+
async denoiseImageData({ imageData, sigma } = {}) {
|
|
116
|
+
validateImageShape(imageData, resolvedInputShape);
|
|
117
|
+
const input = buildAllenkFdncnnInput({ imageData, sigma });
|
|
118
|
+
const result = await this.execute(input);
|
|
119
|
+
return {
|
|
120
|
+
...result,
|
|
121
|
+
imageData: {
|
|
122
|
+
width: imageData.width,
|
|
123
|
+
height: imageData.height,
|
|
124
|
+
data: convertAllenkFdncnnOutputToRgba({
|
|
125
|
+
output: result.output,
|
|
126
|
+
width: imageData.width,
|
|
127
|
+
height: imageData.height
|
|
128
|
+
})
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export {
|
|
136
|
+
createAllenkFdncnnOnnxRuntime,
|
|
137
|
+
validateImageShape
|
|
138
|
+
};
|