@xyo-network/image-thumbnail-plugin 2.71.8 → 2.71.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/Witness/Witness.js +51 -27
- package/dist/cjs/Witness/Witness.js.map +1 -1
- package/dist/docs.json +2178 -1852
- package/dist/esm/Witness/Witness.js +23 -5
- package/dist/esm/Witness/Witness.js.map +1 -1
- package/dist/types/Plugin.d.ts +2 -0
- package/dist/types/Plugin.d.ts.map +1 -1
- package/dist/types/Witness/Config.d.ts +2 -0
- package/dist/types/Witness/Config.d.ts.map +1 -1
- package/dist/types/Witness/Witness.d.ts +4 -1
- package/dist/types/Witness/Witness.d.ts.map +1 -1
- package/package.json +9 -9
- package/src/Witness/Config.ts +2 -0
- package/src/Witness/Witness.ts +51 -29
|
@@ -4,6 +4,7 @@ import { axios } from '@xyo-network/axios';
|
|
|
4
4
|
import { PayloadHasher } from '@xyo-network/core';
|
|
5
5
|
import { ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin';
|
|
6
6
|
import { AbstractWitness } from '@xyo-network/witness';
|
|
7
|
+
import { Semaphore } from 'async-mutex';
|
|
7
8
|
import { subClass } from 'gm';
|
|
8
9
|
import { sync as hasbin } from 'hasbin';
|
|
9
10
|
import { sha256 } from 'hash-wasm';
|
|
@@ -17,8 +18,16 @@ const gm = subClass({ imageMagick: '7+' });
|
|
|
17
18
|
export class ImageThumbnailWitness extends AbstractWitness {
|
|
18
19
|
static configSchemas = [ImageThumbnailWitnessConfigSchema];
|
|
19
20
|
_cache;
|
|
21
|
+
_semaphore = new Semaphore(this.maxAsyncProcesses);
|
|
20
22
|
get cache() {
|
|
21
|
-
this._cache =
|
|
23
|
+
this._cache =
|
|
24
|
+
this._cache ??
|
|
25
|
+
new LRUCache({
|
|
26
|
+
max: this.maxCacheEntries,
|
|
27
|
+
maxSize: this.maxCacheBytes,
|
|
28
|
+
//just returning the size of the data
|
|
29
|
+
sizeCalculation: (value) => value.url?.length ?? 1,
|
|
30
|
+
});
|
|
22
31
|
return this._cache;
|
|
23
32
|
}
|
|
24
33
|
get encoding() {
|
|
@@ -27,6 +36,12 @@ export class ImageThumbnailWitness extends AbstractWitness {
|
|
|
27
36
|
get height() {
|
|
28
37
|
return this.config.height ?? 128;
|
|
29
38
|
}
|
|
39
|
+
get maxAsyncProcesses() {
|
|
40
|
+
return this.config.maxAsyncProcesses ?? 4;
|
|
41
|
+
}
|
|
42
|
+
get maxCacheBytes() {
|
|
43
|
+
return this.config.maxCacheBytes ?? 1024 * 1024 * 64; //64MB max size
|
|
44
|
+
}
|
|
30
45
|
get maxCacheEntries() {
|
|
31
46
|
return this.config.maxCacheEntries ?? 5000;
|
|
32
47
|
}
|
|
@@ -94,7 +109,7 @@ export class ImageThumbnailWitness extends AbstractWitness {
|
|
|
94
109
|
if (!hasbin('magick')) {
|
|
95
110
|
throw Error('ImageMagick is required for this witness');
|
|
96
111
|
}
|
|
97
|
-
return compact(await Promise.all(payloads.map(async ({ url }) => {
|
|
112
|
+
return await this._semaphore.runExclusive(async () => compact(await Promise.all(payloads.map(async ({ url }) => {
|
|
98
113
|
const cachedResult = this.cache.get(url);
|
|
99
114
|
if (cachedResult) {
|
|
100
115
|
return cachedResult;
|
|
@@ -117,9 +132,9 @@ export class ImageThumbnailWitness extends AbstractWitness {
|
|
|
117
132
|
}
|
|
118
133
|
this.cache.set(url, result);
|
|
119
134
|
return result;
|
|
120
|
-
})));
|
|
135
|
+
}))));
|
|
121
136
|
}
|
|
122
|
-
async
|
|
137
|
+
async createThumbnailDataUrl(sourceBuffer) {
|
|
123
138
|
const thumb = await new Promise((resolve, reject) => {
|
|
124
139
|
gm(sourceBuffer)
|
|
125
140
|
.quality(this.quality)
|
|
@@ -188,13 +203,16 @@ export class ImageThumbnailWitness extends AbstractWitness {
|
|
|
188
203
|
if (response.status >= 200 && response.status < 300) {
|
|
189
204
|
const contentType = response.headers['content-type']?.toString();
|
|
190
205
|
if (contentType.split('/')[0] !== 'image') {
|
|
206
|
+
// TODO: Use FFMPEG to create a thumbnail for videos
|
|
207
|
+
// ffmpeg -i input.mp4 -ss 00:00:05 -vframes 1 output.png
|
|
208
|
+
// Then resize thumbnail using
|
|
191
209
|
result.mime = result.mime ?? {};
|
|
192
210
|
result.mime.invalid = true;
|
|
193
211
|
}
|
|
194
212
|
else {
|
|
195
213
|
const sourceBuffer = Buffer.from(response.data, 'binary');
|
|
196
214
|
result.sourceHash = await ImageThumbnailWitness.binaryToSha256(sourceBuffer);
|
|
197
|
-
result.url = await this.
|
|
215
|
+
result.url = await this.createThumbnailDataUrl(sourceBuffer);
|
|
198
216
|
}
|
|
199
217
|
}
|
|
200
218
|
return result;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Witness.js","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,UAAU,CAAA;AAElD,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,KAAK,EAA6B,MAAM,oBAAoB,CAAA;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,OAAO,EAAkB,oBAAoB,EAAE,MAAM,6CAA6C,CAAA;AAElG,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAA;AAC7B,OAAO,EAAE,IAAI,IAAI,MAAM,EAAE,MAAM,QAAQ,CAAA;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,OAAO,MAAM,gBAAgB,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,KAAK,MAAM,QAAQ,CAAA;AAC1B,OAAO,GAAG,MAAM,WAAW,CAAA;AAE3B,OAAO,EAAE,iCAAiC,EAAE,MAAM,UAAU,CAAA;AAG5D,sCAAsC;AAEtC,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;AAW1C,MAAM,OAAO,qBAAiG,SAAQ,eAAwB;IAC5I,MAAM,CAAU,aAAa,GAAG,CAAC,iCAAiC,CAAC,CAAA;IAE3D,MAAM,CAAmC;
|
|
1
|
+
{"version":3,"file":"Witness.js","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,UAAU,CAAA;AAElD,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,KAAK,EAA6B,MAAM,oBAAoB,CAAA;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,OAAO,EAAkB,oBAAoB,EAAE,MAAM,6CAA6C,CAAA;AAElG,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAA;AAC7B,OAAO,EAAE,IAAI,IAAI,MAAM,EAAE,MAAM,QAAQ,CAAA;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,OAAO,MAAM,gBAAgB,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,KAAK,MAAM,QAAQ,CAAA;AAC1B,OAAO,GAAG,MAAM,WAAW,CAAA;AAE3B,OAAO,EAAE,iCAAiC,EAAE,MAAM,UAAU,CAAA;AAG5D,sCAAsC;AAEtC,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;AAW1C,MAAM,OAAO,qBAAiG,SAAQ,eAAwB;IAC5I,MAAM,CAAU,aAAa,GAAG,CAAC,iCAAiC,CAAC,CAAA;IAE3D,MAAM,CAAmC;IACzC,UAAU,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAE1D,IAAI,KAAK;QACP,IAAI,CAAC,MAAM;YACT,IAAI,CAAC,MAAM;gBACX,IAAI,QAAQ,CAAyB;oBACnC,GAAG,EAAE,IAAI,CAAC,eAAe;oBACzB,OAAO,EAAE,IAAI,CAAC,aAAa;oBAC3B,qCAAqC;oBACrC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC;iBACnD,CAAC,CAAA;QACJ,OAAO,IAAI,CAAC,MAAM,CAAA;IACpB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAA;IACtC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAA;IAClC,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,CAAA;IAC3C,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,IAAI,GAAG,IAAI,GAAG,EAAE,CAAA,CAAC,eAAe;IACtE,CAAC;IAED,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,IAAI,CAAA;IAC5C,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAA;IAClC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,GAAG,CAAA;IACjC,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,YAAY,CAAC,UAAkB;QACpC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAA;QAC/B,IAAI,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAA;QAC3B,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,CAAA;QACnB,IAAI,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAA;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAA;QACxB,IAAI,QAAQ,KAAK,OAAO,EAAE;YACxB,QAAQ,GAAG,QAAQ,CAAA;YACnB,IAAI,GAAG,qBAAqB,CAAA;YAC5B,IAAI,GAAG,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,IAAI,EAAE,CAAA;YACtE,MAAM,IAAI,GAAG,GAAG,QAAQ,KAAK,IAAI,IAAI,IAAI,EAAE,CAAA;YAC3C,OAAO,KAAK,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;SACrD;aAAM;YACL,OAAO,UAAU,CAAA;SAClB;IACH,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,IAAgB;QAClD,MAAM,aAAa,CAAC,eAAe,CAAA;QACnC,IAAI,aAAa,CAAC,WAAW,CAAC,UAAU,EAAE;YACxC,IAAI;gBACF,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;aAC1B;YAAC,OAAO,EAAE,EAAE;gBACX,aAAa,CAAC,WAAW,CAAC,SAAS,GAAG,KAAK,CAAA;aAC5C;SACF;QACD,mDAAmD;QACnD,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAA;IACzD,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,GAAW;QAC1C,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAC9B,IAAI,IAAI,EAAE;gBACR,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC/E,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC5C,OAAO,MAAM,CAAA;aACd;iBAAM;gBACL,MAAM,KAAK,GAA+B;oBACxC,OAAO,EAAE,kBAAkB;oBAC3B,IAAI,EAAE,4BAA4B;oBAClC,GAAG;iBACJ,CAAA;gBACD,MAAM,KAAK,CAAA;aACZ;SACF;IACH,CAAC;IAEkB,KAAK,CAAC,cAAc,CAAC,WAAyB,EAAE;QACjE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;YACrB,MAAM,KAAK,CAAC,0CAA0C,CAAC,CAAA;SACxD;QACD,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CACnD,OAAO,CACL,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAA0B,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;YACtD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACxC,IAAI,YAAY,EAAE;gBAChB,OAAO,YAAY,CAAA;aACpB;YACD,IAAI,MAAsB,CAAA;YAE1B,sCAAsC;YACtC,MAAM,UAAU,GAAG,qBAAqB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAA;YAE/D,IAAI,UAAU,EAAE;gBACd,MAAM,GAAG;oBACP,MAAM,EAAE,oBAAoB;oBAC5B,UAAU,EAAE,MAAM,qBAAqB,CAAC,cAAc,CAAC,UAAU,CAAC;oBAClE,SAAS,EAAE,GAAG;oBACd,GAAG;iBACJ,CAAA;aACF;iBAAM;gBACL,uCAAuC;gBACvC,MAAM,UAAU,GAAG,qBAAqB,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;gBAC1D,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;aACzC;YACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YAC3B,OAAO,MAAM,CAAA;QACf,CAAC,CAAC,CACH,CACF,CACF,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,YAAoB;QACvD,MAAM,KAAK,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1D,EAAE,CAAC,YAAY,CAAC;iBACb,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;iBACrB,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;iBAC/B,OAAO,EAAE;iBACT,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACzC,IAAI,KAAK,EAAE;oBACT,MAAM,CAAC,KAAK,CAAC,CAAA;iBACd;qBAAM;oBACL,OAAO,CAAC,MAAM,CAAC,CAAA;iBAChB;YACH,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QACF,OAAO,yBAAyB,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAA;IAC5D,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAAW;QAChC,IAAI,QAAuB,CAAA;QAC3B,IAAI,SAAmB,CAAA;QACvB,IAAI;YACF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;YAC3B,SAAS,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAClD,kEAAkE;SACnE;QAAC,OAAO,EAAE,EAAE;YACX,MAAM,KAAK,GAAG,EAAc,CAAA;YAC5B,MAAM,MAAM,GAAmB;gBAC7B,IAAI,EAAE;oBACJ,QAAQ,EAAE,KAAK,CAAC,IAAI;iBACrB;gBACD,MAAM,EAAE,oBAAoB;gBAC5B,SAAS,EAAE,GAAG;aACf,CAAA;YACD,OAAO,MAAM,CAAA;SACd;QACD,IAAI;YACF,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;gBAC9B,YAAY,EAAE,aAAa;aAC5B,CAAC,CAAA;SACH;QAAC,OAAO,EAAE,EAAE;YACX,MAAM,UAAU,GAAG,EAAgB,CAAA;YACnC,IAAI,UAAU,CAAC,YAAY,EAAE;gBAC3B,yCAAyC;gBACzC,MAAM,MAAM,GAAmB;oBAC7B,IAAI,EAAE;wBACJ,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;wBACvB,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM;qBACrC;oBACD,MAAM,EAAE,oBAAoB;oBAC5B,SAAS,EAAE,GAAG;iBACf,CAAA;gBACD,OAAO,MAAM,CAAA;aACd;iBAAM;gBACL,MAAM,EAAE,CAAA;aACT;SACF;QAED,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE;gBACJ,MAAM,EAAE,QAAQ,CAAC,MAAM;aACxB;YACD,MAAM,EAAE,oBAAoB;YAC5B,SAAS,EAAE,GAAG;SACf,CAAA;QAED,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE;YACnD,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAA;YAChE,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE;gBACzC,oDAAoD;gBACpD,yDAAyD;gBACzD,8BAA8B;gBAC9B,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAA;gBAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;aAC3B;iBAAM;gBACL,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;gBACzD,MAAM,CAAC,UAAU,GAAG,MAAM,qBAAqB,CAAC,cAAc,CAAC,YAAY,CAAC,CAAA;gBAC5E,MAAM,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,CAAA;aAC7D;SACF;QACD,OAAO,MAAM,CAAA;IACf,CAAC"}
|
package/dist/types/Plugin.d.ts
CHANGED
|
@@ -38,6 +38,8 @@ export declare const ImageThumbnailPlugin: () => import("@xyo-network/payloadset
|
|
|
38
38
|
} & {
|
|
39
39
|
encoding?: "PNG" | undefined;
|
|
40
40
|
height?: number | undefined;
|
|
41
|
+
maxAsyncProcesses?: number | undefined;
|
|
42
|
+
maxCacheBytes?: number | undefined;
|
|
41
43
|
maxCacheEntries?: number | undefined;
|
|
42
44
|
quality?: number | undefined;
|
|
43
45
|
schema: "network.xyo.image.thumbnail.witness.config";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Plugin.d.ts","sourceRoot":"","sources":["../../src/Plugin.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAA;AAEjD,eAAO,MAAM,oBAAoB
|
|
1
|
+
{"version":3,"file":"Plugin.d.ts","sourceRoot":"","sources":["../../src/Plugin.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAA;AAEjD,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAS9B,CAAA"}
|
|
@@ -4,6 +4,8 @@ export type ImageThumbnailWitnessConfigSchema = typeof ImageThumbnailWitnessConf
|
|
|
4
4
|
export type ImageThumbnailWitnessConfig = WitnessConfig<{
|
|
5
5
|
encoding?: 'PNG';
|
|
6
6
|
height?: number;
|
|
7
|
+
maxAsyncProcesses?: number;
|
|
8
|
+
maxCacheBytes?: number;
|
|
7
9
|
maxCacheEntries?: number;
|
|
8
10
|
quality?: number;
|
|
9
11
|
schema: ImageThumbnailWitnessConfigSchema;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Config.d.ts","sourceRoot":"","sources":["../../../src/Witness/Config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAEpD,eAAO,MAAM,iCAAiC,8CAAoD,CAAA;AAClG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,MAAM,MAAM,2BAA2B,GAAG,aAAa,CAAC;IACtD,QAAQ,CAAC,EAAE,KAAK,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,iCAAiC,CAAA;IACzC,KAAK,CAAC,EAAE,MAAM,CAAA;CACf,CAAC,CAAA"}
|
|
1
|
+
{"version":3,"file":"Config.d.ts","sourceRoot":"","sources":["../../../src/Witness/Config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAEpD,eAAO,MAAM,iCAAiC,8CAAoD,CAAA;AAClG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,MAAM,MAAM,2BAA2B,GAAG,aAAa,CAAC;IACtD,QAAQ,CAAC,EAAE,KAAK,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,iCAAiC,CAAA;IACzC,KAAK,CAAC,EAAE,MAAM,CAAA;CACf,CAAC,CAAA"}
|
|
@@ -13,6 +13,7 @@ export interface DnsError extends Error {
|
|
|
13
13
|
export declare class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams = ImageThumbnailWitnessParams> extends AbstractWitness<TParams> {
|
|
14
14
|
static configSchemas: "network.xyo.image.thumbnail.witness.config"[];
|
|
15
15
|
private _cache?;
|
|
16
|
+
private _semaphore;
|
|
16
17
|
get cache(): LRUCache<string, import("@xyo-network/payload-model").SchemaFields & import("@xyo-network/payload-model").PayloadFields & {
|
|
17
18
|
http?: {
|
|
18
19
|
dnsError?: string | undefined;
|
|
@@ -31,6 +32,8 @@ export declare class ImageThumbnailWitness<TParams extends ImageThumbnailWitness
|
|
|
31
32
|
}, unknown>;
|
|
32
33
|
get encoding(): "PNG";
|
|
33
34
|
get height(): number;
|
|
35
|
+
get maxAsyncProcesses(): number;
|
|
36
|
+
get maxCacheBytes(): number;
|
|
34
37
|
get maxCacheEntries(): number;
|
|
35
38
|
get quality(): number;
|
|
36
39
|
get width(): number;
|
|
@@ -44,7 +47,7 @@ export declare class ImageThumbnailWitness<TParams extends ImageThumbnailWitness
|
|
|
44
47
|
private static binaryToSha256;
|
|
45
48
|
private static bufferFromDataUrl;
|
|
46
49
|
protected observeHandler(payloads?: UrlPayload[]): Promise<ImageThumbnail[]>;
|
|
47
|
-
private
|
|
50
|
+
private createThumbnailDataUrl;
|
|
48
51
|
private fromHttp;
|
|
49
52
|
}
|
|
50
53
|
//# sourceMappingURL=Witness.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Witness.d.ts","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAwB,MAAM,6CAA6C,CAAA;AAClG,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;
|
|
1
|
+
{"version":3,"file":"Witness.d.ts","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAwB,MAAM,6CAA6C,CAAA;AAClG,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAMtD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAKpC,OAAO,EAAE,2BAA2B,EAAE,MAAM,UAAU,CAAA;AAMtD,MAAM,WAAW,0BAA2B,SAAQ,KAAK;IACvD,IAAI,EAAE,4BAA4B,CAAA;IAClC,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,QAAS,SAAQ,KAAK;IACrC,IAAI,EAAE,MAAM,CAAA;CACb;AAED,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,iDAAsC;IAEnE,OAAO,CAAC,MAAM,CAAC,CAAkC;IACjD,OAAO,CAAC,UAAU,CAAwC;IAE1D,IAAI,KAAK;;;;;;;;;;;;;;;gBAUR;IAED,IAAI,QAAQ,UAEX;IAED,IAAI,MAAM,WAET;IAED,IAAI,iBAAiB,WAEpB;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,eAAe,WAElB;IAED,IAAI,OAAO,WAEV;IAED,IAAI,KAAK,WAER;IAED;;;;;OAKG;IACH,MAAM,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM;mBAiBjB,cAAc;IAanC,OAAO,CAAC,MAAM,CAAC,iBAAiB;cAkBP,cAAc,CAAC,QAAQ,GAAE,UAAU,EAAO,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;YAqCjF,sBAAsB;YAiBtB,QAAQ;CAgEvB"}
|
package/package.json
CHANGED
|
@@ -11,14 +11,14 @@
|
|
|
11
11
|
},
|
|
12
12
|
"dependencies": {
|
|
13
13
|
"@xylabs/url": "^2.9.3",
|
|
14
|
-
"@xyo-network/axios": "~2.71.
|
|
15
|
-
"@xyo-network/core": "~2.71.
|
|
16
|
-
"@xyo-network/image-thumbnail-payload-plugin": "~2.71.
|
|
17
|
-
"@xyo-network/module": "~2.71.
|
|
18
|
-
"@xyo-network/payload-model": "~2.71.
|
|
19
|
-
"@xyo-network/payloadset-plugin": "~2.71.
|
|
20
|
-
"@xyo-network/url-payload-plugin": "~2.71.
|
|
21
|
-
"@xyo-network/witness": "~2.71.
|
|
14
|
+
"@xyo-network/axios": "~2.71.10",
|
|
15
|
+
"@xyo-network/core": "~2.71.10",
|
|
16
|
+
"@xyo-network/image-thumbnail-payload-plugin": "~2.71.10",
|
|
17
|
+
"@xyo-network/module": "~2.71.10",
|
|
18
|
+
"@xyo-network/payload-model": "~2.71.10",
|
|
19
|
+
"@xyo-network/payloadset-plugin": "~2.71.10",
|
|
20
|
+
"@xyo-network/url-payload-plugin": "~2.71.10",
|
|
21
|
+
"@xyo-network/witness": "~2.71.10",
|
|
22
22
|
"gm": "^1.25.0",
|
|
23
23
|
"hasbin": "^1.2.3",
|
|
24
24
|
"hash-wasm": "^4.9.0",
|
|
@@ -68,5 +68,5 @@
|
|
|
68
68
|
},
|
|
69
69
|
"sideEffects": false,
|
|
70
70
|
"types": "dist/types/index.d.ts",
|
|
71
|
-
"version": "2.71.
|
|
71
|
+
"version": "2.71.10"
|
|
72
72
|
}
|
package/src/Witness/Config.ts
CHANGED
|
@@ -7,6 +7,8 @@ export type ImageThumbnailWitnessConfigSchema = typeof ImageThumbnailWitnessConf
|
|
|
7
7
|
export type ImageThumbnailWitnessConfig = WitnessConfig<{
|
|
8
8
|
encoding?: 'PNG'
|
|
9
9
|
height?: number
|
|
10
|
+
maxAsyncProcesses?: number
|
|
11
|
+
maxCacheBytes?: number
|
|
10
12
|
maxCacheEntries?: number
|
|
11
13
|
quality?: number
|
|
12
14
|
schema: ImageThumbnailWitnessConfigSchema
|
package/src/Witness/Witness.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { PayloadHasher } from '@xyo-network/core'
|
|
|
6
6
|
import { ImageThumbnail, ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin'
|
|
7
7
|
import { UrlPayload } from '@xyo-network/url-payload-plugin'
|
|
8
8
|
import { AbstractWitness } from '@xyo-network/witness'
|
|
9
|
+
import { Semaphore } from 'async-mutex'
|
|
9
10
|
import { subClass } from 'gm'
|
|
10
11
|
import { sync as hasbin } from 'hasbin'
|
|
11
12
|
import { sha256 } from 'hash-wasm'
|
|
@@ -34,9 +35,17 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
|
|
|
34
35
|
static override configSchemas = [ImageThumbnailWitnessConfigSchema]
|
|
35
36
|
|
|
36
37
|
private _cache?: LRUCache<string, ImageThumbnail>
|
|
38
|
+
private _semaphore = new Semaphore(this.maxAsyncProcesses)
|
|
37
39
|
|
|
38
40
|
get cache() {
|
|
39
|
-
this._cache =
|
|
41
|
+
this._cache =
|
|
42
|
+
this._cache ??
|
|
43
|
+
new LRUCache<string, ImageThumbnail>({
|
|
44
|
+
max: this.maxCacheEntries,
|
|
45
|
+
maxSize: this.maxCacheBytes,
|
|
46
|
+
//just returning the size of the data
|
|
47
|
+
sizeCalculation: (value) => value.url?.length ?? 1,
|
|
48
|
+
})
|
|
40
49
|
return this._cache
|
|
41
50
|
}
|
|
42
51
|
|
|
@@ -48,6 +57,14 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
|
|
|
48
57
|
return this.config.height ?? 128
|
|
49
58
|
}
|
|
50
59
|
|
|
60
|
+
get maxAsyncProcesses() {
|
|
61
|
+
return this.config.maxAsyncProcesses ?? 4
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
get maxCacheBytes() {
|
|
65
|
+
return this.config.maxCacheBytes ?? 1024 * 1024 * 64 //64MB max size
|
|
66
|
+
}
|
|
67
|
+
|
|
51
68
|
get maxCacheEntries() {
|
|
52
69
|
return this.config.maxCacheEntries ?? 5000
|
|
53
70
|
}
|
|
@@ -118,38 +135,40 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
|
|
|
118
135
|
if (!hasbin('magick')) {
|
|
119
136
|
throw Error('ImageMagick is required for this witness')
|
|
120
137
|
}
|
|
121
|
-
return
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
let result: ImageThumbnail
|
|
129
|
-
|
|
130
|
-
//if it is a data URL, return a Buffer
|
|
131
|
-
const dataBuffer = ImageThumbnailWitness.bufferFromDataUrl(url)
|
|
132
|
-
|
|
133
|
-
if (dataBuffer) {
|
|
134
|
-
result = {
|
|
135
|
-
schema: ImageThumbnailSchema,
|
|
136
|
-
sourceHash: await ImageThumbnailWitness.binaryToSha256(dataBuffer),
|
|
137
|
-
sourceUrl: url,
|
|
138
|
-
url,
|
|
138
|
+
return await this._semaphore.runExclusive(async () =>
|
|
139
|
+
compact(
|
|
140
|
+
await Promise.all(
|
|
141
|
+
payloads.map<Promise<ImageThumbnail>>(async ({ url }) => {
|
|
142
|
+
const cachedResult = this.cache.get(url)
|
|
143
|
+
if (cachedResult) {
|
|
144
|
+
return cachedResult
|
|
139
145
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
let result: ImageThumbnail
|
|
147
|
+
|
|
148
|
+
//if it is a data URL, return a Buffer
|
|
149
|
+
const dataBuffer = ImageThumbnailWitness.bufferFromDataUrl(url)
|
|
150
|
+
|
|
151
|
+
if (dataBuffer) {
|
|
152
|
+
result = {
|
|
153
|
+
schema: ImageThumbnailSchema,
|
|
154
|
+
sourceHash: await ImageThumbnailWitness.binaryToSha256(dataBuffer),
|
|
155
|
+
sourceUrl: url,
|
|
156
|
+
url,
|
|
157
|
+
}
|
|
158
|
+
} else {
|
|
159
|
+
//if it is ipfs, go through cloud flair
|
|
160
|
+
const mutatedUrl = ImageThumbnailWitness.checkIpfsUrl(url)
|
|
161
|
+
result = await this.fromHttp(mutatedUrl)
|
|
162
|
+
}
|
|
163
|
+
this.cache.set(url, result)
|
|
164
|
+
return result
|
|
165
|
+
}),
|
|
166
|
+
),
|
|
148
167
|
),
|
|
149
168
|
)
|
|
150
169
|
}
|
|
151
170
|
|
|
152
|
-
private async
|
|
171
|
+
private async createThumbnailDataUrl(sourceBuffer: Buffer) {
|
|
153
172
|
const thumb = await new Promise<Buffer>((resolve, reject) => {
|
|
154
173
|
gm(sourceBuffer)
|
|
155
174
|
.quality(this.quality)
|
|
@@ -217,12 +236,15 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
|
|
|
217
236
|
if (response.status >= 200 && response.status < 300) {
|
|
218
237
|
const contentType = response.headers['content-type']?.toString()
|
|
219
238
|
if (contentType.split('/')[0] !== 'image') {
|
|
239
|
+
// TODO: Use FFMPEG to create a thumbnail for videos
|
|
240
|
+
// ffmpeg -i input.mp4 -ss 00:00:05 -vframes 1 output.png
|
|
241
|
+
// Then resize thumbnail using
|
|
220
242
|
result.mime = result.mime ?? {}
|
|
221
243
|
result.mime.invalid = true
|
|
222
244
|
} else {
|
|
223
245
|
const sourceBuffer = Buffer.from(response.data, 'binary')
|
|
224
246
|
result.sourceHash = await ImageThumbnailWitness.binaryToSha256(sourceBuffer)
|
|
225
|
-
result.url = await this.
|
|
247
|
+
result.url = await this.createThumbnailDataUrl(sourceBuffer)
|
|
226
248
|
}
|
|
227
249
|
}
|
|
228
250
|
return result
|