@xyo-network/image-thumbnail-plugin 2.71.9 → 2.71.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/Witness/Witness.js +46 -25
- package/dist/cjs/Witness/Witness.js.map +1 -1
- package/dist/docs.json +2174 -1848
- package/dist/esm/Witness/Witness.js +18 -3
- package/dist/esm/Witness/Witness.js.map +1 -1
- package/dist/types/Plugin.d.ts +2 -0
- package/dist/types/Plugin.d.ts.map +1 -1
- package/dist/types/Witness/Config.d.ts +2 -0
- package/dist/types/Witness/Config.d.ts.map +1 -1
- package/dist/types/Witness/Witness.d.ts +3 -0
- package/dist/types/Witness/Witness.d.ts.map +1 -1
- package/package.json +9 -9
- package/src/Witness/Config.ts +2 -0
- package/src/Witness/Witness.ts +46 -27
|
@@ -4,6 +4,7 @@ import { axios } from '@xyo-network/axios';
|
|
|
4
4
|
import { PayloadHasher } from '@xyo-network/core';
|
|
5
5
|
import { ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin';
|
|
6
6
|
import { AbstractWitness } from '@xyo-network/witness';
|
|
7
|
+
import { Semaphore } from 'async-mutex';
|
|
7
8
|
import { subClass } from 'gm';
|
|
8
9
|
import { sync as hasbin } from 'hasbin';
|
|
9
10
|
import { sha256 } from 'hash-wasm';
|
|
@@ -17,8 +18,16 @@ const gm = subClass({ imageMagick: '7+' });
|
|
|
17
18
|
export class ImageThumbnailWitness extends AbstractWitness {
|
|
18
19
|
static configSchemas = [ImageThumbnailWitnessConfigSchema];
|
|
19
20
|
_cache;
|
|
21
|
+
_semaphore = new Semaphore(this.maxAsyncProcesses);
|
|
20
22
|
get cache() {
|
|
21
|
-
this._cache =
|
|
23
|
+
this._cache =
|
|
24
|
+
this._cache ??
|
|
25
|
+
new LRUCache({
|
|
26
|
+
max: this.maxCacheEntries,
|
|
27
|
+
maxSize: this.maxCacheBytes,
|
|
28
|
+
//just returning the size of the data
|
|
29
|
+
sizeCalculation: (value) => value.url?.length ?? 1,
|
|
30
|
+
});
|
|
22
31
|
return this._cache;
|
|
23
32
|
}
|
|
24
33
|
get encoding() {
|
|
@@ -27,6 +36,12 @@ export class ImageThumbnailWitness extends AbstractWitness {
|
|
|
27
36
|
get height() {
|
|
28
37
|
return this.config.height ?? 128;
|
|
29
38
|
}
|
|
39
|
+
get maxAsyncProcesses() {
|
|
40
|
+
return this.config.maxAsyncProcesses ?? 4;
|
|
41
|
+
}
|
|
42
|
+
get maxCacheBytes() {
|
|
43
|
+
return this.config.maxCacheBytes ?? 1024 * 1024 * 64; //64MB max size
|
|
44
|
+
}
|
|
30
45
|
get maxCacheEntries() {
|
|
31
46
|
return this.config.maxCacheEntries ?? 5000;
|
|
32
47
|
}
|
|
@@ -94,7 +109,7 @@ export class ImageThumbnailWitness extends AbstractWitness {
|
|
|
94
109
|
if (!hasbin('magick')) {
|
|
95
110
|
throw Error('ImageMagick is required for this witness');
|
|
96
111
|
}
|
|
97
|
-
return compact(await Promise.all(payloads.map(async ({ url }) => {
|
|
112
|
+
return await this._semaphore.runExclusive(async () => compact(await Promise.all(payloads.map(async ({ url }) => {
|
|
98
113
|
const cachedResult = this.cache.get(url);
|
|
99
114
|
if (cachedResult) {
|
|
100
115
|
return cachedResult;
|
|
@@ -117,7 +132,7 @@ export class ImageThumbnailWitness extends AbstractWitness {
|
|
|
117
132
|
}
|
|
118
133
|
this.cache.set(url, result);
|
|
119
134
|
return result;
|
|
120
|
-
})));
|
|
135
|
+
}))));
|
|
121
136
|
}
|
|
122
137
|
async createThumbnailDataUrl(sourceBuffer) {
|
|
123
138
|
const thumb = await new Promise((resolve, reject) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Witness.js","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,UAAU,CAAA;AAElD,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,KAAK,EAA6B,MAAM,oBAAoB,CAAA;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,OAAO,EAAkB,oBAAoB,EAAE,MAAM,6CAA6C,CAAA;AAElG,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAA;AAC7B,OAAO,EAAE,IAAI,IAAI,MAAM,EAAE,MAAM,QAAQ,CAAA;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,OAAO,MAAM,gBAAgB,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,KAAK,MAAM,QAAQ,CAAA;AAC1B,OAAO,GAAG,MAAM,WAAW,CAAA;AAE3B,OAAO,EAAE,iCAAiC,EAAE,MAAM,UAAU,CAAA;AAG5D,sCAAsC;AAEtC,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;AAW1C,MAAM,OAAO,qBAAiG,SAAQ,eAAwB;IAC5I,MAAM,CAAU,aAAa,GAAG,CAAC,iCAAiC,CAAC,CAAA;IAE3D,MAAM,CAAmC;
|
|
1
|
+
{"version":3,"file":"Witness.js","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,UAAU,CAAA;AAElD,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,KAAK,EAA6B,MAAM,oBAAoB,CAAA;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,OAAO,EAAkB,oBAAoB,EAAE,MAAM,6CAA6C,CAAA;AAElG,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAA;AAC7B,OAAO,EAAE,IAAI,IAAI,MAAM,EAAE,MAAM,QAAQ,CAAA;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClC,OAAO,OAAO,MAAM,gBAAgB,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,KAAK,MAAM,QAAQ,CAAA;AAC1B,OAAO,GAAG,MAAM,WAAW,CAAA;AAE3B,OAAO,EAAE,iCAAiC,EAAE,MAAM,UAAU,CAAA;AAG5D,sCAAsC;AAEtC,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;AAW1C,MAAM,OAAO,qBAAiG,SAAQ,eAAwB;IAC5I,MAAM,CAAU,aAAa,GAAG,CAAC,iCAAiC,CAAC,CAAA;IAE3D,MAAM,CAAmC;IACzC,UAAU,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAE1D,IAAI,KAAK;QACP,IAAI,CAAC,MAAM;YACT,IAAI,CAAC,MAAM;gBACX,IAAI,QAAQ,CAAyB;oBACnC,GAAG,EAAE,IAAI,CAAC,eAAe;oBACzB,OAAO,EAAE,IAAI,CAAC,aAAa;oBAC3B,qCAAqC;oBACrC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC;iBACnD,CAAC,CAAA;QACJ,OAAO,IAAI,CAAC,MAAM,CAAA;IACpB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAA;IACtC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAA;IAClC,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,CAAA;IAC3C,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,IAAI,GAAG,IAAI,GAAG,EAAE,CAAA,CAAC,eAAe;IACtE,CAAC;IAED,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,IAAI,CAAA;IAC5C,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAA;IAClC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,GAAG,CAAA;IACjC,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,YAAY,CAAC,UAAkB;QACpC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAA;QAC/B,IAAI,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAA;QAC3B,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,CAAA;QACnB,IAAI,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAA;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAA;QACxB,IAAI,QAAQ,KAAK,OAAO,EAAE;YACxB,QAAQ,GAAG,QAAQ,CAAA;YACnB,IAAI,GAAG,qBAAqB,CAAA;YAC5B,IAAI,GAAG,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,IAAI,EAAE,CAAA;YACtE,MAAM,IAAI,GAAG,GAAG,QAAQ,KAAK,IAAI,IAAI,IAAI,EAAE,CAAA;YAC3C,OAAO,KAAK,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;SACrD;aAAM;YACL,OAAO,UAAU,CAAA;SAClB;IACH,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,IAAgB;QAClD,MAAM,aAAa,CAAC,eAAe,CAAA;QACnC,IAAI,aAAa,CAAC,WAAW,CAAC,UAAU,EAAE;YACxC,IAAI;gBACF,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;aAC1B;YAAC,OAAO,EAAE,EAAE;gBACX,aAAa,CAAC,WAAW,CAAC,SAAS,GAAG,KAAK,CAAA;aAC5C;SACF;QACD,mDAAmD;QACnD,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAA;IACzD,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,GAAW;QAC1C,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAC9B,IAAI,IAAI,EAAE;gBACR,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC/E,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC5C,OAAO,MAAM,CAAA;aACd;iBAAM;gBACL,MAAM,KAAK,GAA+B;oBACxC,OAAO,EAAE,kBAAkB;oBAC3B,IAAI,EAAE,4BAA4B;oBAClC,GAAG;iBACJ,CAAA;gBACD,MAAM,KAAK,CAAA;aACZ;SACF;IACH,CAAC;IAEkB,KAAK,CAAC,cAAc,CAAC,WAAyB,EAAE;QACjE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;YACrB,MAAM,KAAK,CAAC,0CAA0C,CAAC,CAAA;SACxD;QACD,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CACnD,OAAO,CACL,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAA0B,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;YACtD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACxC,IAAI,YAAY,EAAE;gBAChB,OAAO,YAAY,CAAA;aACpB;YACD,IAAI,MAAsB,CAAA;YAE1B,sCAAsC;YACtC,MAAM,UAAU,GAAG,qBAAqB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAA;YAE/D,IAAI,UAAU,EAAE;gBACd,MAAM,GAAG;oBACP,MAAM,EAAE,oBAAoB;oBAC5B,UAAU,EAAE,MAAM,qBAAqB,CAAC,cAAc,CAAC,UAAU,CAAC;oBAClE,SAAS,EAAE,GAAG;oBACd,GAAG;iBACJ,CAAA;aACF;iBAAM;gBACL,uCAAuC;gBACvC,MAAM,UAAU,GAAG,qBAAqB,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;gBAC1D,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;aACzC;YACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YAC3B,OAAO,MAAM,CAAA;QACf,CAAC,CAAC,CACH,CACF,CACF,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,YAAoB;QACvD,MAAM,KAAK,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1D,EAAE,CAAC,YAAY,CAAC;iBACb,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;iBACrB,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;iBAC/B,OAAO,EAAE;iBACT,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACzC,IAAI,KAAK,EAAE;oBACT,MAAM,CAAC,KAAK,CAAC,CAAA;iBACd;qBAAM;oBACL,OAAO,CAAC,MAAM,CAAC,CAAA;iBAChB;YACH,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QACF,OAAO,yBAAyB,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAA;IAC5D,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAAW;QAChC,IAAI,QAAuB,CAAA;QAC3B,IAAI,SAAmB,CAAA;QACvB,IAAI;YACF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;YAC3B,SAAS,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAClD,kEAAkE;SACnE;QAAC,OAAO,EAAE,EAAE;YACX,MAAM,KAAK,GAAG,EAAc,CAAA;YAC5B,MAAM,MAAM,GAAmB;gBAC7B,IAAI,EAAE;oBACJ,QAAQ,EAAE,KAAK,CAAC,IAAI;iBACrB;gBACD,MAAM,EAAE,oBAAoB;gBAC5B,SAAS,EAAE,GAAG;aACf,CAAA;YACD,OAAO,MAAM,CAAA;SACd;QACD,IAAI;YACF,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;gBAC9B,YAAY,EAAE,aAAa;aAC5B,CAAC,CAAA;SACH;QAAC,OAAO,EAAE,EAAE;YACX,MAAM,UAAU,GAAG,EAAgB,CAAA;YACnC,IAAI,UAAU,CAAC,YAAY,EAAE;gBAC3B,yCAAyC;gBACzC,MAAM,MAAM,GAAmB;oBAC7B,IAAI,EAAE;wBACJ,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;wBACvB,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM;qBACrC;oBACD,MAAM,EAAE,oBAAoB;oBAC5B,SAAS,EAAE,GAAG;iBACf,CAAA;gBACD,OAAO,MAAM,CAAA;aACd;iBAAM;gBACL,MAAM,EAAE,CAAA;aACT;SACF;QAED,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE;gBACJ,MAAM,EAAE,QAAQ,CAAC,MAAM;aACxB;YACD,MAAM,EAAE,oBAAoB;YAC5B,SAAS,EAAE,GAAG;SACf,CAAA;QAED,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE;YACnD,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAA;YAChE,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE;gBACzC,oDAAoD;gBACpD,yDAAyD;gBACzD,8BAA8B;gBAC9B,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAA;gBAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;aAC3B;iBAAM;gBACL,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;gBACzD,MAAM,CAAC,UAAU,GAAG,MAAM,qBAAqB,CAAC,cAAc,CAAC,YAAY,CAAC,CAAA;gBAC5E,MAAM,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,CAAA;aAC7D;SACF;QACD,OAAO,MAAM,CAAA;IACf,CAAC"}
|
package/dist/types/Plugin.d.ts
CHANGED
|
@@ -38,6 +38,8 @@ export declare const ImageThumbnailPlugin: () => import("@xyo-network/payloadset
|
|
|
38
38
|
} & {
|
|
39
39
|
encoding?: "PNG" | undefined;
|
|
40
40
|
height?: number | undefined;
|
|
41
|
+
maxAsyncProcesses?: number | undefined;
|
|
42
|
+
maxCacheBytes?: number | undefined;
|
|
41
43
|
maxCacheEntries?: number | undefined;
|
|
42
44
|
quality?: number | undefined;
|
|
43
45
|
schema: "network.xyo.image.thumbnail.witness.config";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Plugin.d.ts","sourceRoot":"","sources":["../../src/Plugin.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAA;AAEjD,eAAO,MAAM,oBAAoB
|
|
1
|
+
{"version":3,"file":"Plugin.d.ts","sourceRoot":"","sources":["../../src/Plugin.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAA;AAEjD,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAS9B,CAAA"}
|
|
@@ -4,6 +4,8 @@ export type ImageThumbnailWitnessConfigSchema = typeof ImageThumbnailWitnessConf
|
|
|
4
4
|
export type ImageThumbnailWitnessConfig = WitnessConfig<{
|
|
5
5
|
encoding?: 'PNG';
|
|
6
6
|
height?: number;
|
|
7
|
+
maxAsyncProcesses?: number;
|
|
8
|
+
maxCacheBytes?: number;
|
|
7
9
|
maxCacheEntries?: number;
|
|
8
10
|
quality?: number;
|
|
9
11
|
schema: ImageThumbnailWitnessConfigSchema;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Config.d.ts","sourceRoot":"","sources":["../../../src/Witness/Config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAEpD,eAAO,MAAM,iCAAiC,8CAAoD,CAAA;AAClG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,MAAM,MAAM,2BAA2B,GAAG,aAAa,CAAC;IACtD,QAAQ,CAAC,EAAE,KAAK,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,iCAAiC,CAAA;IACzC,KAAK,CAAC,EAAE,MAAM,CAAA;CACf,CAAC,CAAA"}
|
|
1
|
+
{"version":3,"file":"Config.d.ts","sourceRoot":"","sources":["../../../src/Witness/Config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAEpD,eAAO,MAAM,iCAAiC,8CAAoD,CAAA;AAClG,MAAM,MAAM,iCAAiC,GAAG,OAAO,iCAAiC,CAAA;AAExF,MAAM,MAAM,2BAA2B,GAAG,aAAa,CAAC;IACtD,QAAQ,CAAC,EAAE,KAAK,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,iCAAiC,CAAA;IACzC,KAAK,CAAC,EAAE,MAAM,CAAA;CACf,CAAC,CAAA"}
|
|
@@ -13,6 +13,7 @@ export interface DnsError extends Error {
|
|
|
13
13
|
export declare class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams = ImageThumbnailWitnessParams> extends AbstractWitness<TParams> {
|
|
14
14
|
static configSchemas: "network.xyo.image.thumbnail.witness.config"[];
|
|
15
15
|
private _cache?;
|
|
16
|
+
private _semaphore;
|
|
16
17
|
get cache(): LRUCache<string, import("@xyo-network/payload-model").SchemaFields & import("@xyo-network/payload-model").PayloadFields & {
|
|
17
18
|
http?: {
|
|
18
19
|
dnsError?: string | undefined;
|
|
@@ -31,6 +32,8 @@ export declare class ImageThumbnailWitness<TParams extends ImageThumbnailWitness
|
|
|
31
32
|
}, unknown>;
|
|
32
33
|
get encoding(): "PNG";
|
|
33
34
|
get height(): number;
|
|
35
|
+
get maxAsyncProcesses(): number;
|
|
36
|
+
get maxCacheBytes(): number;
|
|
34
37
|
get maxCacheEntries(): number;
|
|
35
38
|
get quality(): number;
|
|
36
39
|
get width(): number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Witness.d.ts","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAwB,MAAM,6CAA6C,CAAA;AAClG,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;
|
|
1
|
+
{"version":3,"file":"Witness.d.ts","sourceRoot":"","sources":["../../../src/Witness/Witness.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAwB,MAAM,6CAA6C,CAAA;AAClG,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAMtD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAKpC,OAAO,EAAE,2BAA2B,EAAE,MAAM,UAAU,CAAA;AAMtD,MAAM,WAAW,0BAA2B,SAAQ,KAAK;IACvD,IAAI,EAAE,4BAA4B,CAAA;IAClC,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,QAAS,SAAQ,KAAK;IACrC,IAAI,EAAE,MAAM,CAAA;CACb;AAED,qBAAa,qBAAqB,CAAC,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,CAAE,SAAQ,eAAe,CAAC,OAAO,CAAC;IAC5I,OAAgB,aAAa,iDAAsC;IAEnE,OAAO,CAAC,MAAM,CAAC,CAAkC;IACjD,OAAO,CAAC,UAAU,CAAwC;IAE1D,IAAI,KAAK;;;;;;;;;;;;;;;gBAUR;IAED,IAAI,QAAQ,UAEX;IAED,IAAI,MAAM,WAET;IAED,IAAI,iBAAiB,WAEpB;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,eAAe,WAElB;IAED,IAAI,OAAO,WAEV;IAED,IAAI,KAAK,WAER;IAED;;;;;OAKG;IACH,MAAM,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM;mBAiBjB,cAAc;IAanC,OAAO,CAAC,MAAM,CAAC,iBAAiB;cAkBP,cAAc,CAAC,QAAQ,GAAE,UAAU,EAAO,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;YAqCjF,sBAAsB;YAiBtB,QAAQ;CAgEvB"}
|
package/package.json
CHANGED
|
@@ -11,14 +11,14 @@
|
|
|
11
11
|
},
|
|
12
12
|
"dependencies": {
|
|
13
13
|
"@xylabs/url": "^2.9.3",
|
|
14
|
-
"@xyo-network/axios": "~2.71.
|
|
15
|
-
"@xyo-network/core": "~2.71.
|
|
16
|
-
"@xyo-network/image-thumbnail-payload-plugin": "~2.71.
|
|
17
|
-
"@xyo-network/module": "~2.71.
|
|
18
|
-
"@xyo-network/payload-model": "~2.71.
|
|
19
|
-
"@xyo-network/payloadset-plugin": "~2.71.
|
|
20
|
-
"@xyo-network/url-payload-plugin": "~2.71.
|
|
21
|
-
"@xyo-network/witness": "~2.71.
|
|
14
|
+
"@xyo-network/axios": "~2.71.10",
|
|
15
|
+
"@xyo-network/core": "~2.71.10",
|
|
16
|
+
"@xyo-network/image-thumbnail-payload-plugin": "~2.71.10",
|
|
17
|
+
"@xyo-network/module": "~2.71.10",
|
|
18
|
+
"@xyo-network/payload-model": "~2.71.10",
|
|
19
|
+
"@xyo-network/payloadset-plugin": "~2.71.10",
|
|
20
|
+
"@xyo-network/url-payload-plugin": "~2.71.10",
|
|
21
|
+
"@xyo-network/witness": "~2.71.10",
|
|
22
22
|
"gm": "^1.25.0",
|
|
23
23
|
"hasbin": "^1.2.3",
|
|
24
24
|
"hash-wasm": "^4.9.0",
|
|
@@ -68,5 +68,5 @@
|
|
|
68
68
|
},
|
|
69
69
|
"sideEffects": false,
|
|
70
70
|
"types": "dist/types/index.d.ts",
|
|
71
|
-
"version": "2.71.
|
|
71
|
+
"version": "2.71.10"
|
|
72
72
|
}
|
package/src/Witness/Config.ts
CHANGED
|
@@ -7,6 +7,8 @@ export type ImageThumbnailWitnessConfigSchema = typeof ImageThumbnailWitnessConf
|
|
|
7
7
|
export type ImageThumbnailWitnessConfig = WitnessConfig<{
|
|
8
8
|
encoding?: 'PNG'
|
|
9
9
|
height?: number
|
|
10
|
+
maxAsyncProcesses?: number
|
|
11
|
+
maxCacheBytes?: number
|
|
10
12
|
maxCacheEntries?: number
|
|
11
13
|
quality?: number
|
|
12
14
|
schema: ImageThumbnailWitnessConfigSchema
|
package/src/Witness/Witness.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { PayloadHasher } from '@xyo-network/core'
|
|
|
6
6
|
import { ImageThumbnail, ImageThumbnailSchema } from '@xyo-network/image-thumbnail-payload-plugin'
|
|
7
7
|
import { UrlPayload } from '@xyo-network/url-payload-plugin'
|
|
8
8
|
import { AbstractWitness } from '@xyo-network/witness'
|
|
9
|
+
import { Semaphore } from 'async-mutex'
|
|
9
10
|
import { subClass } from 'gm'
|
|
10
11
|
import { sync as hasbin } from 'hasbin'
|
|
11
12
|
import { sha256 } from 'hash-wasm'
|
|
@@ -34,9 +35,17 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
|
|
|
34
35
|
static override configSchemas = [ImageThumbnailWitnessConfigSchema]
|
|
35
36
|
|
|
36
37
|
private _cache?: LRUCache<string, ImageThumbnail>
|
|
38
|
+
private _semaphore = new Semaphore(this.maxAsyncProcesses)
|
|
37
39
|
|
|
38
40
|
get cache() {
|
|
39
|
-
this._cache =
|
|
41
|
+
this._cache =
|
|
42
|
+
this._cache ??
|
|
43
|
+
new LRUCache<string, ImageThumbnail>({
|
|
44
|
+
max: this.maxCacheEntries,
|
|
45
|
+
maxSize: this.maxCacheBytes,
|
|
46
|
+
//just returning the size of the data
|
|
47
|
+
sizeCalculation: (value) => value.url?.length ?? 1,
|
|
48
|
+
})
|
|
40
49
|
return this._cache
|
|
41
50
|
}
|
|
42
51
|
|
|
@@ -48,6 +57,14 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
|
|
|
48
57
|
return this.config.height ?? 128
|
|
49
58
|
}
|
|
50
59
|
|
|
60
|
+
get maxAsyncProcesses() {
|
|
61
|
+
return this.config.maxAsyncProcesses ?? 4
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
get maxCacheBytes() {
|
|
65
|
+
return this.config.maxCacheBytes ?? 1024 * 1024 * 64 //64MB max size
|
|
66
|
+
}
|
|
67
|
+
|
|
51
68
|
get maxCacheEntries() {
|
|
52
69
|
return this.config.maxCacheEntries ?? 5000
|
|
53
70
|
}
|
|
@@ -118,33 +135,35 @@ export class ImageThumbnailWitness<TParams extends ImageThumbnailWitnessParams =
|
|
|
118
135
|
if (!hasbin('magick')) {
|
|
119
136
|
throw Error('ImageMagick is required for this witness')
|
|
120
137
|
}
|
|
121
|
-
return
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
let result: ImageThumbnail
|
|
129
|
-
|
|
130
|
-
//if it is a data URL, return a Buffer
|
|
131
|
-
const dataBuffer = ImageThumbnailWitness.bufferFromDataUrl(url)
|
|
132
|
-
|
|
133
|
-
if (dataBuffer) {
|
|
134
|
-
result = {
|
|
135
|
-
schema: ImageThumbnailSchema,
|
|
136
|
-
sourceHash: await ImageThumbnailWitness.binaryToSha256(dataBuffer),
|
|
137
|
-
sourceUrl: url,
|
|
138
|
-
url,
|
|
138
|
+
return await this._semaphore.runExclusive(async () =>
|
|
139
|
+
compact(
|
|
140
|
+
await Promise.all(
|
|
141
|
+
payloads.map<Promise<ImageThumbnail>>(async ({ url }) => {
|
|
142
|
+
const cachedResult = this.cache.get(url)
|
|
143
|
+
if (cachedResult) {
|
|
144
|
+
return cachedResult
|
|
139
145
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
let result: ImageThumbnail
|
|
147
|
+
|
|
148
|
+
//if it is a data URL, return a Buffer
|
|
149
|
+
const dataBuffer = ImageThumbnailWitness.bufferFromDataUrl(url)
|
|
150
|
+
|
|
151
|
+
if (dataBuffer) {
|
|
152
|
+
result = {
|
|
153
|
+
schema: ImageThumbnailSchema,
|
|
154
|
+
sourceHash: await ImageThumbnailWitness.binaryToSha256(dataBuffer),
|
|
155
|
+
sourceUrl: url,
|
|
156
|
+
url,
|
|
157
|
+
}
|
|
158
|
+
} else {
|
|
159
|
+
//if it is ipfs, go through cloud flair
|
|
160
|
+
const mutatedUrl = ImageThumbnailWitness.checkIpfsUrl(url)
|
|
161
|
+
result = await this.fromHttp(mutatedUrl)
|
|
162
|
+
}
|
|
163
|
+
this.cache.set(url, result)
|
|
164
|
+
return result
|
|
165
|
+
}),
|
|
166
|
+
),
|
|
148
167
|
),
|
|
149
168
|
)
|
|
150
169
|
}
|