@prosopo/datasets-fs 0.1.17 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +7 -8
- package/dist/cli.js.map +1 -1
- package/dist/flatten/cli.js +3 -4
- package/dist/flatten/cli.js.map +1 -1
- package/dist/flatten/flatten.js +2 -3
- package/dist/flatten/flatten.js.map +1 -1
- package/dist/generate/cli.js +2 -3
- package/dist/generate/cli.js.map +1 -1
- package/dist/generate/distinct/cli.js +3 -4
- package/dist/generate/distinct/cli.js.map +1 -1
- package/dist/generate/distinct/generate.d.ts.map +1 -1
- package/dist/generate/distinct/generate.js +12 -12
- package/dist/generate/distinct/generate.js.map +1 -1
- package/dist/generate/union/cli.js +3 -4
- package/dist/generate/union/cli.js.map +1 -1
- package/dist/generate/union/generate.d.ts.map +1 -1
- package/dist/generate/union/generate.js +11 -11
- package/dist/generate/union/generate.js.map +1 -1
- package/dist/generate/util.d.ts +0 -6
- package/dist/generate/util.d.ts.map +1 -1
- package/dist/generate/util.js +0 -20
- package/dist/generate/util.js.map +1 -1
- package/dist/get/cli.js +3 -4
- package/dist/get/cli.js.map +1 -1
- package/dist/get/get.d.ts.map +1 -1
- package/dist/get/get.js +12 -10
- package/dist/get/get.js.map +1 -1
- package/dist/labels/cli.js +3 -4
- package/dist/labels/cli.js.map +1 -1
- package/dist/labels/labels.js +2 -3
- package/dist/labels/labels.js.map +1 -1
- package/dist/relocate/cli.js +3 -4
- package/dist/relocate/cli.js.map +1 -1
- package/dist/relocate/relocate.d.ts.map +1 -1
- package/dist/relocate/relocate.js +8 -6
- package/dist/relocate/relocate.js.map +1 -1
- package/dist/scale/cli.js +3 -4
- package/dist/scale/cli.js.map +1 -1
- package/dist/scale/scale.js +10 -8
- package/dist/scale/scale.js.map +1 -1
- package/package.json +10 -5
- package/src/cli.ts +0 -50
- package/src/flatten/args.ts +0 -11
- package/src/flatten/cli.ts +0 -33
- package/src/flatten/flatten.ts +0 -77
- package/src/flatten/index.ts +0 -3
- package/src/generate/args.ts +0 -18
- package/src/generate/cli.ts +0 -65
- package/src/generate/distinct/args.ts +0 -13
- package/src/generate/distinct/cli.ts +0 -34
- package/src/generate/distinct/dummy.ts +0 -35
- package/src/generate/distinct/generate.ts +0 -238
- package/src/generate/distinct/index.ts +0 -3
- package/src/generate/index.ts +0 -4
- package/src/generate/union/args.ts +0 -14
- package/src/generate/union/cli.ts +0 -38
- package/src/generate/union/generate.ts +0 -210
- package/src/generate/union/index.ts +0 -3
- package/src/generate/util.ts +0 -61
- package/src/get/args.ts +0 -9
- package/src/get/cli.ts +0 -21
- package/src/get/get.ts +0 -56
- package/src/get/index.ts +0 -3
- package/src/index.ts +0 -14
- package/src/labels/args.ts +0 -9
- package/src/labels/cli.ts +0 -21
- package/src/labels/index.ts +0 -3
- package/src/labels/labels.ts +0 -27
- package/src/relocate/args.ts +0 -11
- package/src/relocate/cli.ts +0 -32
- package/src/relocate/relocate.ts +0 -38
- package/src/scale/args.ts +0 -13
- package/src/scale/cli.ts +0 -43
- package/src/scale/index.ts +0 -3
- package/src/scale/scale.ts +0 -79
- package/tsconfig.json +0 -17
- package/tsconfig.tsbuildinfo +0 -1
package/dist/cli.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { LogLevelSchema, getLogger } from '@prosopo/common';
|
|
1
|
+
import { LogLevel, getLogger } from '@prosopo/common';
|
|
3
2
|
import { hideBin } from 'yargs/helpers';
|
|
4
3
|
import esMain from 'es-main';
|
|
5
4
|
import flatten from './flatten/cli.js';
|
|
@@ -11,14 +10,14 @@ import relocate from './relocate/cli.js';
|
|
|
11
10
|
import scale from './scale/cli.js';
|
|
12
11
|
import yargs from 'yargs';
|
|
13
12
|
const dirname = process.cwd();
|
|
14
|
-
const logger = getLogger(
|
|
15
|
-
const main = () =>
|
|
16
|
-
|
|
13
|
+
const logger = getLogger(LogLevel.enum.info, `${dirname}`);
|
|
14
|
+
const main = async () => {
|
|
15
|
+
await yargs(hideBin(process.argv))
|
|
17
16
|
.help()
|
|
18
17
|
.option('log-level', {
|
|
19
18
|
type: 'string',
|
|
20
|
-
choices: Object.values(
|
|
21
|
-
default:
|
|
19
|
+
choices: Object.values(LogLevel.options),
|
|
20
|
+
default: LogLevel.enum.info,
|
|
22
21
|
description: 'The log level',
|
|
23
22
|
})
|
|
24
23
|
.middleware((argv) => {
|
|
@@ -34,7 +33,7 @@ const main = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
34
33
|
.showHelpOnFail(false, 'Specify --help for available options')
|
|
35
34
|
.fail(false)
|
|
36
35
|
.parse();
|
|
37
|
-
}
|
|
36
|
+
};
|
|
38
37
|
//if main process
|
|
39
38
|
if (esMain(import.meta)) {
|
|
40
39
|
main()
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AACvC,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,OAAO,MAAM,kBAAkB,CAAA;AACtC,OAAO,QAAQ,MAAM,mBAAmB,CAAA;AACxC,OAAO,GAAG,MAAM,cAAc,CAAA;AAC9B,OAAO,MAAM,MAAM,iBAAiB,CAAA;AACpC,OAAO,OAAO,MAAM,SAAS,CAAA;AAC7B,OAAO,QAAQ,MAAM,mBAAmB,CAAA;AACxC,OAAO,KAAK,MAAM,gBAAgB,CAAA;AAClC,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;AAC7B,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;AAE1D,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;IACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC7B,IAAI,EAAE;SACN,MAAM,CAAC,WAAW,EAAE;QACjB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;QACxC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI;QAC3B,WAAW,EAAE,eAAe;KAC/B,CAAC;SACD,UAAU,CAAC,CAAC,IAAS,EAAE,EAAE;QACtB,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACrC,CAAC,CAAC;SACD,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC7B,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC5B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC1B,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC7B,OAAO,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SACxB,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC3B,cAAc,EAAE;SAChB,cAAc,CAAC,KAAK,EAAE,sCAAsC,CAAC;SAC7D,IAAI,CAAC,KAAK,CAAC;SACX,KAAK,EAAE,CAAA;AAChB,CAAC,CAAA;AAED,iBAAiB;AACjB,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;IACrB,IAAI,EAAE;SACD,IAAI,CAAC,GAAG,EAAE;QACP,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAC3B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;CACT"}
|
package/dist/flatten/cli.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import { argsSchema } from './args.js';
|
|
3
2
|
import flatten from './flatten.js';
|
|
4
3
|
export default (cmdArgs) => {
|
|
@@ -23,9 +22,9 @@ export default (cmdArgs) => {
|
|
|
23
22
|
description: 'Overwrite the output file if it already exists',
|
|
24
23
|
});
|
|
25
24
|
},
|
|
26
|
-
handler: (argv) =>
|
|
27
|
-
|
|
28
|
-
}
|
|
25
|
+
handler: async (argv) => {
|
|
26
|
+
await flatten(argsSchema.parse(argv), cmdArgs?.logger);
|
|
27
|
+
},
|
|
29
28
|
};
|
|
30
29
|
};
|
|
31
30
|
//# sourceMappingURL=cli.js.map
|
package/dist/flatten/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/flatten/cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/flatten/cli.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,OAAO,MAAM,cAAc,CAAA;AAElC,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,SAAS;QAClB,QAAQ,EACJ,wIAAwI;QAC5I,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,oFAAoF;aACpG,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,mFAAmF;aACnG,CAAC;iBACD,MAAM,CAAC,WAAW,EAAE;gBACjB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,gDAAgD;aAChE,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAC1D,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}
|
package/dist/flatten/flatten.js
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import { CaptchaItemTypes, DataSchema } from '@prosopo/types';
|
|
3
2
|
import { ProsopoEnvError, getLoggerDefault } from '@prosopo/common';
|
|
4
3
|
import { blake2b } from '@noble/hashes/blake2b';
|
|
5
4
|
import { u8aToHex } from '@polkadot/util';
|
|
6
5
|
import fs from 'fs';
|
|
7
|
-
export default (args, logger) =>
|
|
6
|
+
export default async (args, logger) => {
|
|
8
7
|
logger = logger || getLoggerDefault();
|
|
9
8
|
logger.debug(args, 'flattening...');
|
|
10
9
|
const dataDir = args.data;
|
|
@@ -62,5 +61,5 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
62
61
|
DataSchema.parse(data);
|
|
63
62
|
// write map file
|
|
64
63
|
fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4));
|
|
65
|
-
}
|
|
64
|
+
};
|
|
66
65
|
//# sourceMappingURL=flatten.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flatten.js","sourceRoot":"","sources":["../../src/flatten/flatten.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"flatten.js","sourceRoot":"","sources":["../../src/flatten/flatten.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAQ,UAAU,EAAgB,MAAM,gBAAgB,CAAA;AACjF,OAAO,EAAU,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAA;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AACzC,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,eAAe,KAAK,EAAE,IAAU,EAAE,MAAe,EAAE,EAAE;IACjD,MAAM,GAAG,MAAM,IAAI,gBAAgB,EAAE,CAAA;IAErC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IAEnC,MAAM,OAAO,GAAW,IAAI,CAAC,IAAI,CAAA;IACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QACzB,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,kCAAkC,OAAO,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAA;KAC9G;IACD,MAAM,MAAM,GAAW,IAAI,CAAC,GAAG,CAAA;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAA;IACzC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;QACrC,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,oCAAoC,MAAM,EAAE,CAAC,EACvD,6BAA6B,CAChC,CAAA;KACJ;IAED,yEAAyE;IACzE,MAAM,MAAM,GAAa,EAAE;SACtB,WAAW,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;SAC7C,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;SACxC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAEjC,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,GAAG,MAAM,SAAS,CAAA;IACnC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,iBAAiB;IACjB,MAAM,KAAK,GAAmB,EAAE,CAAA;IAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QACxB,sBAAsB;QACtB,MAAM,MAAM,GAAa,EAAE,CAAC,WAAW,CAAC,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CAAA;QAC9D,iBAAiB;QACjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YACxB,MAAM,CAAC,GAAG,CAAC,cAAc,KAAK,IAAI,KAAK,EAAE,CAAC,CAAA;YAC1C,yCAAyC;YACzC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAA;YACxC,qBAAqB;YACrB,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC,CAAA;YAC/D,yCAAyC;YACzC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;YAC7B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;YAC1B,MAAM,IAAI,GAAG,GAAG,GAAG,IAAI,SAAS,EAAE,CAAA;YAClC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,EAAE;gBACtC,MAAM,CAAC,GAAG,CAAC,oBAAoB,KAAK,IAAI,KAAK,OAAO,IAAI,EAAE,CAAC,CAAA;aAC9D;YACD,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,EAAE,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAA;YACtE,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAA;YACvD,gCAAgC;YAChC,MAAM,KAAK,GAAiB;gBACxB,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,gBAAgB,CAAC,KAAK;gBAC5B,KAAK;gBACL,IAAI,EAAE,GAAG;aACZ,CAAA;YACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;SACpB;KACJ;IAED,MAAM,IAAI,GAAS;QACf,KAAK;KACR,CAAA;IAED,cAAc;IACd,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEtB,iBAAiB;IACjB,EAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;AAC1E,CAAC,CAAA"}
|
package/dist/generate/cli.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import generateDistinct from './distinct/cli.js';
|
|
3
2
|
import generateUnion from './union/cli.js';
|
|
4
3
|
export default (cmdArgs) => {
|
|
@@ -55,9 +54,9 @@ export default (cmdArgs) => {
|
|
|
55
54
|
description: 'Allow duplicates in the unlabelled data',
|
|
56
55
|
});
|
|
57
56
|
},
|
|
58
|
-
handler: (argv) =>
|
|
57
|
+
handler: async (argv) => {
|
|
59
58
|
throw new Error('Please specify a command');
|
|
60
|
-
}
|
|
59
|
+
},
|
|
61
60
|
};
|
|
62
61
|
};
|
|
63
62
|
//# sourceMappingURL=cli.js.map
|
package/dist/generate/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/generate/cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/generate/cli.ts"],"names":[],"mappings":"AAEA,OAAO,gBAAgB,MAAM,mBAAmB,CAAA;AAChD,OAAO,aAAa,MAAM,gBAAgB,CAAA;AAE1C,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,UAAU;QACnB,QAAQ,EAAE,mBAAmB;QAC7B,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;iBAClC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;iBAC/B,aAAa,EAAE;iBACf,MAAM,CAAC,WAAW,EAAE;gBACjB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,gDAAgD;aAChE,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,yBAAyB;gBACtC,OAAO,EAAE,eAAe;aAC3B,CAAC;iBACD,MAAM,CAAC,UAAU,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,0DAA0D;aAC1E,CAAC;iBACD,MAAM,CAAC,YAAY,EAAE;gBAClB,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,qEAAqE;aACrF,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,sCAAsC;aACtD,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,kCAAkC;aAClD,CAAC;iBACD,OAAO,CAAC,QAAQ,EAAE;gBACf,IAAI,EAAE,QAAQ;gBACd,WAAW,EACP,gHAAgH;aACvH,CAAC;iBACD,MAAM,CAAC,kBAAkB,EAAE;gBACxB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,wDAAwD;aACxE,CAAC;iBACD,MAAM,CAAC,2BAA2B,EAAE;gBACjC,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,uCAAuC;aACvD,CAAC;iBACD,MAAM,CAAC,6BAA6B,EAAE;gBACnC,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,yCAAyC;aACzD,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAC/C,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import { argsSchema } from './args.js';
|
|
3
2
|
import generate from './generate.js';
|
|
4
3
|
export default (cmdArgs) => {
|
|
@@ -24,9 +23,9 @@ export default (cmdArgs) => {
|
|
|
24
23
|
description: 'Maximum number of target images in each captcha',
|
|
25
24
|
});
|
|
26
25
|
},
|
|
27
|
-
handler: (argv) =>
|
|
28
|
-
|
|
29
|
-
}
|
|
26
|
+
handler: async (argv) => {
|
|
27
|
+
await generate(argsSchema.parse(argv), cmdArgs?.logger);
|
|
28
|
+
},
|
|
30
29
|
};
|
|
31
30
|
};
|
|
32
31
|
//# sourceMappingURL=cli.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../../src/generate/distinct/cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../../src/generate/distinct/cli.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,QAAQ,MAAM,eAAe,CAAA;AAEpC,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,UAAU;QACnB,QAAQ,EACJ,8GAA8G;QAClH,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,MAAM,CAAC,QAAQ,EAAE;gBACd,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,gDAAgD;aAChE,CAAC;iBACD,MAAM,CAAC,UAAU,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,kDAAkD;aAClE,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,iDAAiD;aACjE,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,iDAAiD;aACjE,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAC3D,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/generate/distinct/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAahC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+BAQ/C,IAAI,WAAW,MAAM;AAAjD,
|
|
1
|
+
{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/generate/distinct/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAahC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+BAQ/C,IAAI,WAAW,MAAM;AAAjD,wBAyNC"}
|
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import { CaptchaTypes, CaptchasContainerSchema, DataSchema, LabelledDataSchema, LabelsContainerSchema, } from '@prosopo/types';
|
|
3
2
|
import { ProsopoEnvError, getLoggerDefault } from '@prosopo/common';
|
|
3
|
+
import { at, get, lodash, setSeedGlobal } from '@prosopo/util';
|
|
4
4
|
import { blake2AsHex } from '@polkadot/util-crypto';
|
|
5
5
|
import { checkDuplicates } from '../util.js';
|
|
6
|
-
import { lodash, setSeedGlobal } from '@prosopo/util';
|
|
7
6
|
import bcrypt from 'bcrypt';
|
|
8
7
|
import cliProgress from 'cli-progress';
|
|
9
8
|
import fs from 'fs';
|
|
10
|
-
export default (args, logger) =>
|
|
9
|
+
export default async (args, logger) => {
|
|
11
10
|
logger = logger || getLoggerDefault();
|
|
12
11
|
logger.debug(args, 'generating...');
|
|
13
12
|
const outFile = args.out;
|
|
@@ -52,8 +51,9 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
52
51
|
// split the labelled data by label
|
|
53
52
|
const labelToImages = {};
|
|
54
53
|
for (const entry of labelled) {
|
|
55
|
-
|
|
56
|
-
|
|
54
|
+
const arr = labelToImages[entry.label] || [];
|
|
55
|
+
arr.push(entry);
|
|
56
|
+
labelToImages[entry.label] = arr;
|
|
57
57
|
}
|
|
58
58
|
const targets = Object.keys(labelToImages);
|
|
59
59
|
// load the labels from file
|
|
@@ -80,14 +80,14 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
80
80
|
throw new ProsopoEnvError(new Error(`not enough different labels in labelled data: ${labelledMapFile}`), 'DATASET.NOT_ENOUGH_LABELS');
|
|
81
81
|
}
|
|
82
82
|
// uniformly sample targets
|
|
83
|
-
const target = targets
|
|
83
|
+
const target = at(targets, i % targets.length);
|
|
84
84
|
const notTargets = targets.filter((t) => t !== target);
|
|
85
85
|
// how many correct items should be in the captcha?
|
|
86
86
|
const nCorrect = _.random(minCorrect, maxCorrect);
|
|
87
87
|
// how many incorrect items should be in the captcha?
|
|
88
88
|
const nIncorrect = size - nCorrect;
|
|
89
|
-
const targetItems = labelToImages
|
|
90
|
-
const notTargetItems = notTargets.map((notTarget) => labelToImages
|
|
89
|
+
const targetItems = get(labelToImages, target);
|
|
90
|
+
const notTargetItems = notTargets.map((notTarget) => get(labelToImages, notTarget)).flat();
|
|
91
91
|
if (targetItems.length < nCorrect) {
|
|
92
92
|
throw new ProsopoEnvError(new Error(`not enough images for target (${target})`), 'DATASET.NOT_ENOUGH_IMAGES');
|
|
93
93
|
}
|
|
@@ -101,7 +101,7 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
101
101
|
let items = [...correctItems, ...incorrectItems];
|
|
102
102
|
let indices = [...Array(items.length).keys()];
|
|
103
103
|
indices = _.shuffle(indices);
|
|
104
|
-
items = indices.map((i) => items
|
|
104
|
+
items = indices.map((i) => at(items, i));
|
|
105
105
|
items = items.map((item) => {
|
|
106
106
|
return {
|
|
107
107
|
data: item.data,
|
|
@@ -149,14 +149,14 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
149
149
|
throw new ProsopoEnvError(new Error(`no labels found for unlabelled data: ${labelsFile}`), 'DATASET.NOT_ENOUGH_LABELS');
|
|
150
150
|
}
|
|
151
151
|
const index = _.random(0, labels.length - 1);
|
|
152
|
-
const target = labels
|
|
152
|
+
const target = at(labels, index);
|
|
153
153
|
// randomly pick images from the unlabelled data
|
|
154
154
|
const itemSet = _.sampleSize(unlabelled, size);
|
|
155
155
|
// shuffle the items
|
|
156
156
|
let items = [...itemSet];
|
|
157
157
|
let indices = [...Array(items.length).keys()];
|
|
158
158
|
indices = _.shuffle(indices);
|
|
159
|
-
items = indices.map((i) => items
|
|
159
|
+
items = indices.map((i) => at(items, i));
|
|
160
160
|
items = items.map((item) => {
|
|
161
161
|
return {
|
|
162
162
|
data: item.data,
|
|
@@ -182,5 +182,5 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
182
182
|
// verify the output
|
|
183
183
|
CaptchasContainerSchema.parse(output);
|
|
184
184
|
fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
|
|
185
|
-
}
|
|
185
|
+
};
|
|
186
186
|
//# sourceMappingURL=generate.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../../src/generate/distinct/generate.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../../src/generate/distinct/generate.ts"],"names":[],"mappings":"AACA,OAAO,EACH,YAAY,EAGZ,uBAAuB,EACvB,UAAU,EAEV,kBAAkB,EAElB,qBAAqB,GAExB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAU,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,MAAM,MAAM,QAAQ,CAAA;AAC3B,OAAO,WAAW,MAAM,cAAc,CAAA;AACtC,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,eAAe,KAAK,EAAE,IAAU,EAAE,MAAe,EAAE,EAAE;IACjD,MAAM,GAAG,MAAM,IAAI,gBAAgB,EAAE,CAAA;IAErC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IAEnC,MAAM,OAAO,GAAW,IAAI,CAAC,GAAG,CAAA;IAChC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAA;IACzC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QACtC,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,+BAA+B,OAAO,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAA;KAC3G;IACD,MAAM,eAAe,GAAuB,IAAI,CAAC,QAAQ,CAAA;IACzD,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QACpD,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,qCAAqC,eAAe,EAAE,CAAC,EACjE,mBAAmB,CACtB,CAAA;KACJ;IACD,MAAM,iBAAiB,GAAuB,IAAI,CAAC,UAAU,CAAA;IAC7D,IAAI,iBAAiB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE;QACxD,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,uCAAuC,iBAAiB,EAAE,CAAC,EACrE,mBAAmB,CACtB,CAAA;KACJ;IACD,MAAM,UAAU,GAAuB,IAAI,CAAC,MAAM,CAAA;IAClD,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;IACnC,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;IACnC,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,IAAI,CAAC,CAAA;IAC/C,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,IAAI,IAAI,GAAG,CAAC,CAAA;IACtD,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM,IAAI,CAAC,CAAA;IACvC,MAAM,QAAQ,GAAW,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;IAC3C,MAAM,UAAU,GAAG,EAAE,CAAA;IACrB,MAAM,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAA;IAC7F,MAAM,yBAAyB,GAAG,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAA;IAEjG,eAAe;IACf,aAAa,CAAC,IAAI,CAAC,CAAA;IACnB,+BAA+B;IAC/B,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAElB,uDAAuD;IACvD,MAAM,QAAQ,GAAmB,eAAe;QAC5C,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;QACtF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,UAAU,GAAW,iBAAiB;QACxC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;QAChF,CAAC,CAAC,EAAE,CAAA;IAER,uBAAuB;IACvB,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE;QAClC,uBAAuB;QACvB,yBAAyB;KAC5B,CAAC,CAAA;IAEF,mCAAmC;IACnC,MAAM,aAAa,GAAgC,EAAE,CAAA;IACrD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAC5C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACf,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAA;KACnC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAE1C,4BAA4B;IAC5B,gEAAgE;IAChE,+IAA+I;IAC/I,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,UAAU,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QACzC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;KAC3G;SAAM;QACH,kDAAkD;QAClD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;KAC/B;IAED,6BAA6B;IAC7B,MAAM,cAAc,GAAuB,EAAE,CAAA;IAC7C,kEAAkE;IAClE,MAAM,SAAS,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IAEnF,MAAM,CAAC,IAAI,CAAC,cAAc,MAAM,qBAAqB,CAAC,CAAA;IACtD,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE;QAC7B,iDAAiD;QACjD,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAEvB,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE;YACrB,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,iDAAiD,eAAe,EAAE,CAAC,EAC7E,2BAA2B,CAC9B,CAAA;SACJ;QAED,2BAA2B;QAC3B,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAA;QAEtD,mDAAmD;QACnD,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;QACjD,qDAAqD;QACrD,MAAM,UAAU,GAAG,IAAI,GAAG,QAAQ,CAAA;QAElC,MAAM,WAAW,GAAW,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;QACtD,MAAM,cAAc,GAAW,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAElG,IAAI,WAAW,CAAC,MAAM,GAAG,QAAQ,EAAE;YAC/B,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,iCAAiC,MAAM,GAAG,CAAC,EACrD,2BAA2B,CAC9B,CAAA;SACJ;QACD,IAAI,cAAc,CAAC,MAAM,GAAG,UAAU,EAAE;YACpC,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,8CAA8C,MAAM,GAAG,CAAC,EAClE,2BAA2B,CAC9B,CAAA;SACJ;QAED,wBAAwB;QACxB,MAAM,YAAY,GAAW,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;QAEhE,0BAA0B;QAC1B,MAAM,cAAc,GAAW,CAAC,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAAC,CAAA;QAEvE,IAAI,KAAK,GAAW,CAAC,GAAG,YAAY,EAAE,GAAG,cAAc,CAAC,CAAA;QACxD,IAAI,OAAO,GAAa,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACvD,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAC5B,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;QACxC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvB,OAAO;gBACH,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;aAClB,CAAA;QACL,CAAC,CAAC,CAAA;QAEF,4CAA4C;QAC5C,MAAM,QAAQ,GAAkB,OAAO;aAClC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YACd,OAAO;gBACH,GAAG,EAAE,KAAK;gBACV,IAAI,EAAE,CAAC,EAAE,8CAA8C;aAC1D,CAAA;QACL,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,0GAA0G;aAC3J,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,OAAO,IAAI,CAAC,IAAI,CAAA,CAAC,yCAAyC;QAC9D,CAAC,CAAC,CAAA;QAEN,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAA;QACxD,qBAAqB;QACrB,MAAM,OAAO,GAAqB;YAC9B,IAAI;YACJ,MAAM;YACN,KAAK;YACL,QAAQ;SACX,CAAA;QACD,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;KAC/B;IACD,SAAS,CAAC,IAAI,EAAE,CAAA;IAChB,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,uBAAuB,CAAC,CAAA;IAC1D,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IACrF,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAC9B,+BAA+B;IAC/B,MAAM,gBAAgB,GAAuB,EAAE,CAAA;IAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE;QAC/B,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACzB,IAAI,UAAU,CAAC,MAAM,IAAI,IAAI,EAAE;YAC3B,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,qDAAqD,iBAAiB,EAAE,CAAC,EACnF,2BAA2B,CAC9B,CAAA;SACJ;QACD,uCAAuC;QACvC,wEAAwE;QACxE,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE;YACpB,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,wCAAwC,UAAU,EAAE,CAAC,EAC/D,2BAA2B,CAC9B,CAAA;SACJ;QACD,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAC5C,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;QAChC,gDAAgD;QAChD,MAAM,OAAO,GAAW,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;QACtD,oBAAoB;QACpB,IAAI,KAAK,GAAW,CAAC,GAAG,OAAO,CAAC,CAAA;QAChC,IAAI,OAAO,GAAa,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACvD,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAC5B,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;QACxC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvB,OAAO;gBACH,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;aAClB,CAAA;QACL,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAA;QACxD,qBAAqB;QACrB,MAAM,OAAO,GAAqB;YAC9B,IAAI;YACJ,MAAM;YACN,KAAK;SACR,CAAA;QACD,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;KACjC;IACD,WAAW,CAAC,IAAI,EAAE,CAAA;IAClB,gBAAgB;IAChB,MAAM,MAAM,GAAa;QACrB,QAAQ,EAAE,CAAC,GAAG,cAAc,EAAE,GAAG,gBAAgB,CAAC;QAClD,MAAM,EAAE,YAAY,CAAC,SAAS;KACjC,CAAA;IAED,oBAAoB;IACpB,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAErC,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC,CAAA"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import { argsSchema } from './args.js';
|
|
3
2
|
import generate from './generate.js';
|
|
4
3
|
export default (cmdArgs) => {
|
|
@@ -28,9 +27,9 @@ export default (cmdArgs) => {
|
|
|
28
27
|
description: 'Maximum number of labelled images in each captcha',
|
|
29
28
|
});
|
|
30
29
|
},
|
|
31
|
-
handler: (argv) =>
|
|
32
|
-
|
|
33
|
-
}
|
|
30
|
+
handler: async (argv) => {
|
|
31
|
+
await generate(argsSchema.parse(argv), cmdArgs?.logger);
|
|
32
|
+
},
|
|
34
33
|
};
|
|
35
34
|
};
|
|
36
35
|
//# sourceMappingURL=cli.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../../src/generate/union/cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../../src/generate/union/cli.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,QAAQ,MAAM,eAAe,CAAA;AAEpC,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,OAAO;QAChB,QAAQ,EACJ,gJAAgJ;QACpJ,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,MAAM,CAAC,OAAO,EAAE;gBACb,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,gCAAgC;aAChD,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,iDAAiD;aACjE,CAAC;iBACD,MAAM,CAAC,eAAe,EAAE;gBACrB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,oDAAoD;aACpE,CAAC;iBACD,MAAM,CAAC,cAAc,EAAE;gBACpB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,mDAAmD;aACnE,CAAC;iBACD,MAAM,CAAC,cAAc,EAAE;gBACpB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,mDAAmD;aACnE,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAC3D,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/generate/union/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAahC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+BAO/C,IAAI,WAAW,MAAM;AAAjD,
|
|
1
|
+
{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/generate/union/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAahC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+BAO/C,IAAI,WAAW,MAAM;AAAjD,wBA8LC"}
|
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import { CaptchaTypes, CaptchasContainerSchema, DataSchema, LabelledDataSchema, LabelsContainerSchema, } from '@prosopo/types';
|
|
3
2
|
import { ProsopoEnvError, getLoggerDefault } from '@prosopo/common';
|
|
3
|
+
import { at, get, lodash, setSeedGlobal } from '@prosopo/util';
|
|
4
4
|
import { blake2AsHex } from '@polkadot/util-crypto';
|
|
5
5
|
import { checkDuplicates } from '../util.js';
|
|
6
|
-
import { lodash, setSeedGlobal } from '@prosopo/util';
|
|
7
6
|
import bcrypt from 'bcrypt';
|
|
8
7
|
import fs from 'fs';
|
|
9
|
-
export default (args, logger) =>
|
|
8
|
+
export default async (args, logger) => {
|
|
10
9
|
logger = logger || getLoggerDefault();
|
|
11
10
|
logger.debug(args, 'generating...');
|
|
12
11
|
const outFile = args.out;
|
|
@@ -56,8 +55,9 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
56
55
|
// split the labelled data by label
|
|
57
56
|
const labelToImages = {};
|
|
58
57
|
for (const entry of labelled) {
|
|
59
|
-
|
|
60
|
-
|
|
58
|
+
const arr = labelToImages[entry.label] || [];
|
|
59
|
+
arr.push(entry);
|
|
60
|
+
labelToImages[entry.label] = arr;
|
|
61
61
|
}
|
|
62
62
|
const targets = Object.keys(labelToImages);
|
|
63
63
|
// load the labels from file
|
|
@@ -79,7 +79,7 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
79
79
|
throw new ProsopoEnvError(new Error(`not enough different labels in labelled data: ${labelledMapFile}`), 'DATASET.NOT_ENOUGH_LABELS');
|
|
80
80
|
}
|
|
81
81
|
// uniformly sample targets
|
|
82
|
-
const target = targets
|
|
82
|
+
const target = at(targets, i % targets.length);
|
|
83
83
|
const notTargets = targets.filter((t) => t !== target);
|
|
84
84
|
// how many labelled images should be in the captcha?
|
|
85
85
|
const nLabelled = _.random(minLabelled, maxLabelled);
|
|
@@ -88,8 +88,8 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
88
88
|
const nCorrect = _.random(minCorrect, maxCorrect);
|
|
89
89
|
const nIncorrect = nLabelled - nCorrect;
|
|
90
90
|
const nUnlabelled = size - nLabelled;
|
|
91
|
-
const targetItems = labelToImages
|
|
92
|
-
const notTargetItems = notTargets.map((notTarget) => labelToImages
|
|
91
|
+
const targetItems = get(labelToImages, target);
|
|
92
|
+
const notTargetItems = notTargets.map((notTarget) => get(labelToImages, notTarget)).flat();
|
|
93
93
|
if (nUnlabelled > unlabelled.length) {
|
|
94
94
|
throw new ProsopoEnvError(new Error(`not enough unlabelled data`), 'DATASET.NOT_ENOUGH_IMAGES');
|
|
95
95
|
}
|
|
@@ -107,13 +107,13 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
107
107
|
const unlabelledItems = new Set();
|
|
108
108
|
while (unlabelledItems.size < size - nLabelled) {
|
|
109
109
|
// get a random image from the unlabelled data
|
|
110
|
-
const image = unlabelled
|
|
110
|
+
const image = at(unlabelled, _.random(0, unlabelled.length - 1));
|
|
111
111
|
unlabelledItems.add(image);
|
|
112
112
|
}
|
|
113
113
|
let items = [...correctItems, ...incorrectItems, ...unlabelledItems];
|
|
114
114
|
let indices = [...Array(items.length).keys()];
|
|
115
115
|
indices = _.shuffle(indices);
|
|
116
|
-
items = indices.map((i) => items
|
|
116
|
+
items = indices.map((i) => at(items, i));
|
|
117
117
|
items = items.map((item) => {
|
|
118
118
|
return {
|
|
119
119
|
data: item.data,
|
|
@@ -164,5 +164,5 @@ export default (args, logger) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
164
164
|
// verify the output
|
|
165
165
|
CaptchasContainerSchema.parse(output);
|
|
166
166
|
fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
|
|
167
|
-
}
|
|
167
|
+
};
|
|
168
168
|
//# sourceMappingURL=generate.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../../src/generate/union/generate.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../../src/generate/union/generate.ts"],"names":[],"mappings":"AACA,OAAO,EACH,YAAY,EAGZ,uBAAuB,EACvB,UAAU,EAEV,kBAAkB,EAElB,qBAAqB,GAExB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAU,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,MAAM,MAAM,QAAQ,CAAA;AAC3B,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,eAAe,KAAK,EAAE,IAAU,EAAE,MAAe,EAAE,EAAE;IACjD,MAAM,GAAG,MAAM,IAAI,gBAAgB,EAAE,CAAA;IAErC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IAEnC,MAAM,OAAO,GAAW,IAAI,CAAC,GAAG,CAAA;IAChC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAA;IACzC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QACtC,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,+BAA+B,OAAO,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAA;KAC3G;IACD,MAAM,eAAe,GAAuB,IAAI,CAAC,QAAQ,CAAA;IACzD,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QACpD,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,qCAAqC,eAAe,EAAE,CAAC,EACjE,mBAAmB,CACtB,CAAA;KACJ;IACD,MAAM,iBAAiB,GAAuB,IAAI,CAAC,UAAU,CAAA;IAC7D,IAAI,iBAAiB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE;QACxD,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,uCAAuC,iBAAiB,EAAE,CAAC,EACrE,mBAAmB,CACtB,CAAA;KACJ;IACD,MAAM,UAAU,GAAuB,IAAI,CAAC,MAAM,CAAA;IAClD,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;IACnC,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;IACnC,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,IAAI,CAAC,CAAA;IAC/C,MAAM,UAAU,GAAG,EAAE,CAAA;IACrB,MAAM,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAA;IAC7F,MAAM,yBAAyB,GAAG,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAA;IACjG,MAAM,YAAY,GAAW,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA,CAAC,6BAA6B;IAC9F,MAAM,WAAW,GAAW,UAAU,GAAG,YAAY,CAAA,CAAC,0BAA0B;IAChF,MAAM,WAAW,GAAW,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE,IAAI,CAAC,CAAA,CAAC,4BAA4B;IACjG,MAAM,KAAK,GAAW,IAAI,CAAC,KAAK,IAAI,CAAC,CAAA;IAErC,eAAe;IACf,aAAa,CAAC,IAAI,CAAC,CAAA;IACnB,+BAA+B;IAC/B,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAElB,gUAAgU;IAChU,uKAAuK;IACvK,+TAA+T;IAC/T,6jBAA6jB;IAC7jB,uDAAuD;IACvD,MAAM,QAAQ,GAAmB,eAAe;QAC5C,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;QACtF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,UAAU,GAAW,iBAAiB;QACxC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;QAChF,CAAC,CAAC,EAAE,CAAA;IACR,uBAAuB;IACvB,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE;QAClC,uBAAuB;QACvB,yBAAyB;KAC5B,CAAC,CAAA;IACF,mCAAmC;IACnC,MAAM,aAAa,GAAgC,EAAE,CAAA;IACrD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAC5C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACf,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAA;KACnC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAC1C,4BAA4B;IAC5B,gEAAgE;IAChE,8IAA8I;IAC9I,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,UAAU,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QACzC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;KAC3G;SAAM;QACH,6CAA6C;QAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;KAC/B;IACD,sBAAsB;IACtB,MAAM,QAAQ,GAAuB,EAAE,CAAA;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,OAAO,KAAK,EAAE,CAAC,CAAA;QAEtD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE;YACrB,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,iDAAiD,eAAe,EAAE,CAAC,EAC7E,2BAA2B,CAC9B,CAAA;SACJ;QAED,2BAA2B;QAC3B,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAA;QACtD,qDAAqD;QACrD,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;QACpD,6DAA6D;QAC7D,MAAM,UAAU,GAAG,SAAS,GAAG,UAAU,CAAA;QACzC,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;QACjD,MAAM,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAA;QACvC,MAAM,WAAW,GAAG,IAAI,GAAG,SAAS,CAAA;QAEpC,MAAM,WAAW,GAAG,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;QAC9C,MAAM,cAAc,GAAW,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAElG,IAAI,WAAW,GAAG,UAAU,CAAC,MAAM,EAAE;YACjC,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE,2BAA2B,CAAC,CAAA;SAClG;QACD,IAAI,QAAQ,GAAG,WAAW,CAAC,MAAM,EAAE;YAC/B,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,iCAAiC,MAAM,GAAG,CAAC,EACrD,2BAA2B,CAC9B,CAAA;SACJ;QACD,IAAI,UAAU,GAAG,cAAc,CAAC,MAAM,EAAE;YACpC,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,8CAA8C,MAAM,GAAG,CAAC,EAClE,2BAA2B,CAC9B,CAAA;SACJ;QAED,wBAAwB;QACxB,MAAM,YAAY,GAAW,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;QAEhE,0BAA0B;QAC1B,MAAM,cAAc,GAAW,CAAC,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAAC,CAAA;QAEvE,2BAA2B;QAC3B,MAAM,eAAe,GAAG,IAAI,GAAG,EAAQ,CAAA;QACvC,OAAO,eAAe,CAAC,IAAI,GAAG,IAAI,GAAG,SAAS,EAAE;YAC5C,8CAA8C;YAC9C,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;YAChE,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;SAC7B;QAED,IAAI,KAAK,GAAW,CAAC,GAAG,YAAY,EAAE,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC,CAAA;QAC5E,IAAI,OAAO,GAAa,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACvD,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAC5B,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;QACxC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvB,OAAO;gBACH,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;aAClB,CAAA;QACL,CAAC,CAAC,CAAA;QAEF,4CAA4C;QAC5C,MAAM,QAAQ,GAAkB,OAAO;aAClC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YACd,OAAO;gBACH,GAAG,EAAE,KAAK;gBACV,IAAI,EAAE,CAAC,EAAE,8CAA8C;aAC1D,CAAA;QACL,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,0GAA0G;aAC3J,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,OAAO,IAAI,CAAC,IAAI,CAAA,CAAC,yCAAyC;QAC9D,CAAC,CAAC,CAAA;QAEN,8DAA8D;QAC9D,MAAM,iBAAiB,GAAkB,OAAO;aAC3C,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YACd,OAAO;gBACH,GAAG,EAAE,KAAK;gBACV,IAAI,EAAE,CAAC,EAAE,8CAA8C;aAC1D,CAAA;QACL,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,YAAY,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,0GAA0G;aACpL,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,OAAO,IAAI,CAAC,IAAI,CAAA,CAAC,yCAAyC;QAC9D,CAAC,CAAC,CAAA;QAEN,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAA;QACxD,qBAAqB;QACrB,MAAM,OAAO,GAAqB;YAC9B,IAAI;YACJ,MAAM;YACN,KAAK;YACL,QAAQ;YACR,UAAU,EAAE,iBAAiB;SAChC,CAAA;QACD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;KACzB;IACD,gBAAgB;IAChB,MAAM,MAAM,GAAa;QACrB,QAAQ;QACR,MAAM,EAAE,YAAY,CAAC,SAAS;KACjC,CAAA;IAED,oBAAoB;IACpB,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAErC,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC,CAAA"}
|
package/dist/generate/util.d.ts
CHANGED
|
@@ -3,10 +3,4 @@ export declare const checkDuplicates: (labelled: LabelledItem[], unlabelled: Ite
|
|
|
3
3
|
allowDuplicatesLabelled?: boolean;
|
|
4
4
|
allowDuplicatesUnlabelled?: boolean;
|
|
5
5
|
}) => void;
|
|
6
|
-
export declare const choice: <T>(items: T[], n: number, random: () => number, options?: {
|
|
7
|
-
withReplacement?: boolean;
|
|
8
|
-
}) => {
|
|
9
|
-
choices: T[];
|
|
10
|
-
indices: number[];
|
|
11
|
-
};
|
|
12
6
|
//# sourceMappingURL=util.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/generate/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAEnD,eAAO,MAAM,eAAe,aACd,YAAY,EAAE,cACZ,IAAI,EAAE,WACT;IACL,uBAAuB,CAAC,EAAE,OAAO,CAAA;IACjC,yBAAyB,CAAC,EAAE,OAAO,CAAA;CACtC,SAmBJ,CAAA
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/generate/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAEnD,eAAO,MAAM,eAAe,aACd,YAAY,EAAE,cACZ,IAAI,EAAE,WACT;IACL,uBAAuB,CAAC,EAAE,OAAO,CAAA;IACjC,yBAAyB,CAAC,EAAE,OAAO,CAAA;CACtC,SAmBJ,CAAA"}
|
package/dist/generate/util.js
CHANGED
|
@@ -17,24 +17,4 @@ export const checkDuplicates = (labelled, unlabelled, options) => {
|
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
19
|
};
|
|
20
|
-
export const choice = (items, n, random, options) => {
|
|
21
|
-
if (n > items.length) {
|
|
22
|
-
throw new Error(`n (${n}) cannot be greater than items.length (${items.length})`);
|
|
23
|
-
}
|
|
24
|
-
options = options || {};
|
|
25
|
-
const indicesSet = new Set();
|
|
26
|
-
const indices = [];
|
|
27
|
-
while (indices.length < n) {
|
|
28
|
-
const index = Math.abs(Math.round(random())) % items.length;
|
|
29
|
-
// with replacement == allow duplicates
|
|
30
|
-
// without replacement == don't allow duplicates
|
|
31
|
-
if (options.withReplacement || indicesSet.add(index)) {
|
|
32
|
-
indices.push(index);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
return {
|
|
36
|
-
choices: indices.map((index) => items[index]),
|
|
37
|
-
indices,
|
|
38
|
-
};
|
|
39
|
-
};
|
|
40
20
|
//# sourceMappingURL=util.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/generate/util.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,eAAe,GAAG,CAC3B,QAAwB,EACxB,UAAkB,EAClB,OAGC,EACH,EAAE;IACA,uBAAuB;IACvB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE;QAClC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;YAC1B,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBACrB,MAAM,IAAI,KAAK,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;aACrF;YACD,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;SACtB;KACJ;IACD,IAAI,CAAC,OAAO,CAAC,yBAAyB,EAAE;QACpC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;YAC5B,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBACrB,MAAM,IAAI,KAAK,CAAC,4CAA4C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;aACvF;SACJ;KACJ;AACL,CAAC,CAAA
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/generate/util.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,eAAe,GAAG,CAC3B,QAAwB,EACxB,UAAkB,EAClB,OAGC,EACH,EAAE;IACA,uBAAuB;IACvB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE;QAClC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;YAC1B,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBACrB,MAAM,IAAI,KAAK,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;aACrF;YACD,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;SACtB;KACJ;IACD,IAAI,CAAC,OAAO,CAAC,yBAAyB,EAAE;QACpC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;YAC5B,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBACrB,MAAM,IAAI,KAAK,CAAC,4CAA4C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;aACvF;SACJ;KACJ;AACL,CAAC,CAAA"}
|
package/dist/get/cli.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import { argsSchema } from './args.js';
|
|
3
2
|
import get from './get.js';
|
|
4
3
|
export default (cmdArgs) => {
|
|
@@ -12,9 +11,9 @@ export default (cmdArgs) => {
|
|
|
12
11
|
description: 'JSON file containing urls under a "data" key',
|
|
13
12
|
});
|
|
14
13
|
},
|
|
15
|
-
handler: (argv) =>
|
|
16
|
-
|
|
17
|
-
}
|
|
14
|
+
handler: async (argv) => {
|
|
15
|
+
await get(argsSchema.parse(argv), cmdArgs?.logger);
|
|
16
|
+
},
|
|
18
17
|
};
|
|
19
18
|
};
|
|
20
19
|
//# sourceMappingURL=cli.js.map
|
package/dist/get/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/get/cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/get/cli.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,GAAG,MAAM,UAAU,CAAA;AAE1B,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,KAAK;QACd,QAAQ,EAAE,kCAAkC;QAC5C,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE;gBACxB,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,8CAA8C;aAC9D,CAAC,CAAA;QACN,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QACtD,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}
|
package/dist/get/get.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get.d.ts","sourceRoot":"","sources":["../../src/get/get.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+
|
|
1
|
+
{"version":3,"file":"get.d.ts","sourceRoot":"","sources":["../../src/get/get.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+BAM/C,IAAI,cAAc,MAAM;AAApD,wBAmDC"}
|
package/dist/get/get.js
CHANGED
|
@@ -1,23 +1,25 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import { ProsopoEnvError, getLoggerDefault } from '@prosopo/common';
|
|
2
|
+
import { get } from '@prosopo/util';
|
|
3
|
+
import { z } from 'zod';
|
|
3
4
|
import fetch from 'node-fetch';
|
|
4
5
|
import fs from 'fs';
|
|
5
|
-
export default (args, loggerOpt) =>
|
|
6
|
+
export default async (args, loggerOpt) => {
|
|
6
7
|
const logger = loggerOpt || getLoggerDefault();
|
|
7
8
|
logger.debug(args, 'getting...');
|
|
8
|
-
const traverse = (data) =>
|
|
9
|
+
const traverse = async (data) => {
|
|
9
10
|
if (data instanceof Array) {
|
|
10
11
|
for (let i = 0; i < data.length; i++) {
|
|
11
|
-
data[i] =
|
|
12
|
+
data[i] = await traverse(data[i]);
|
|
12
13
|
}
|
|
13
14
|
}
|
|
14
15
|
else if (data instanceof Object) {
|
|
15
16
|
for (const key of Object.keys(data)) {
|
|
16
17
|
if (key == 'data') {
|
|
17
|
-
const
|
|
18
|
+
const value = get(data, key);
|
|
19
|
+
const url = z.string().parse(value);
|
|
18
20
|
if (url.startsWith('http')) {
|
|
19
21
|
try {
|
|
20
|
-
const response =
|
|
22
|
+
const response = await fetch(url);
|
|
21
23
|
if (!response.ok) {
|
|
22
24
|
logger.error(`GET ${url} ${response.status} ${response.statusText}`);
|
|
23
25
|
}
|
|
@@ -41,18 +43,18 @@ export default (args, loggerOpt) => __awaiter(void 0, void 0, void 0, function*
|
|
|
41
43
|
}
|
|
42
44
|
}
|
|
43
45
|
else {
|
|
44
|
-
|
|
46
|
+
await traverse(get(data, key));
|
|
45
47
|
}
|
|
46
48
|
}
|
|
47
49
|
}
|
|
48
50
|
return data;
|
|
49
|
-
}
|
|
51
|
+
};
|
|
50
52
|
const file = args.data;
|
|
51
53
|
if (!fs.existsSync(file)) {
|
|
52
54
|
throw new ProsopoEnvError(new Error(`file does not exist: ${file}`), 'FS.FILE_NOT_FOUND');
|
|
53
55
|
}
|
|
54
56
|
// read the map file
|
|
55
57
|
const data = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
56
|
-
|
|
57
|
-
}
|
|
58
|
+
await traverse(data);
|
|
59
|
+
};
|
|
58
60
|
//# sourceMappingURL=get.js.map
|
package/dist/get/get.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get.js","sourceRoot":"","sources":["../../src/get/get.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"get.js","sourceRoot":"","sources":["../../src/get/get.ts"],"names":[],"mappings":"AACA,OAAO,EAAU,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,eAAe,KAAK,EAAE,IAAU,EAAE,SAAkB,EAAE,EAAE;IACpD,MAAM,MAAM,GAAG,SAAS,IAAI,gBAAgB,EAAE,CAAA;IAE9C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IAEhC,MAAM,QAAQ,GAAG,KAAK,EAAE,IAAS,EAAE,EAAE;QACjC,IAAI,IAAI,YAAY,KAAK,EAAE;YACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;aACpC;SACJ;aAAM,IAAI,IAAI,YAAY,MAAM,EAAE;YAC/B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACjC,IAAI,GAAG,IAAI,MAAM,EAAE;oBACf,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;oBAC5B,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;oBACnC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;wBACxB,IAAI;4BACA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAA;4BACjC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE;gCACd,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;6BACvE;iCAAM;gCACH,MAAM,CAAC,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC,CAAA;6BAC9B;yBACJ;wBAAC,OAAO,GAAG,EAAE;4BACV,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;yBACpB;qBACJ;yBAAM;wBACH,kBAAkB;wBAClB,IAAI;4BACA,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;4BACpB,MAAM,CAAC,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC,CAAA;yBAC9B;wBAAC,OAAO,GAAG,EAAE;4BACV,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC,CAAA;yBACpC;qBACJ;iBACJ;qBAAM;oBACH,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAA;iBACjC;aACJ;SACJ;QACD,OAAO,IAAI,CAAA;IACf,CAAC,CAAA;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;IACtB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;QACtB,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,EAAE,mBAAmB,CAAC,CAAA;KAC5F;IAED,oBAAoB;IACpB,MAAM,IAAI,GAAQ,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;IAC3D,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC,CAAA"}
|
package/dist/labels/cli.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { __awaiter } from "tslib";
|
|
2
1
|
import { argsSchema } from './args.js';
|
|
3
2
|
import labels from './labels.js';
|
|
4
3
|
export default (cmdArgs) => {
|
|
@@ -12,9 +11,9 @@ export default (cmdArgs) => {
|
|
|
12
11
|
description: 'JSON file containing data',
|
|
13
12
|
});
|
|
14
13
|
},
|
|
15
|
-
handler: (argv) =>
|
|
16
|
-
|
|
17
|
-
}
|
|
14
|
+
handler: async (argv) => {
|
|
15
|
+
await labels(argsSchema.parse(argv), cmdArgs?.logger);
|
|
16
|
+
},
|
|
18
17
|
};
|
|
19
18
|
};
|
|
20
19
|
//# sourceMappingURL=cli.js.map
|
package/dist/labels/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/labels/cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/labels/cli.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,MAAM,MAAM,aAAa,CAAA;AAEhC,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,QAAQ;QACjB,QAAQ,EAAE,+BAA+B;QACzC,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE;gBACxB,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,2BAA2B;aAC3C,CAAC,CAAA;QACN,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QACzD,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}
|