@prosopo/datasets-fs 0.2.6 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/README.md +21 -0
  2. package/dist/cjs/cli/cli.cjs +55 -0
  3. package/dist/cjs/index.cjs +3 -1
  4. package/dist/cli/cli.d.ts +9 -0
  5. package/dist/cli/cli.d.ts.map +1 -0
  6. package/dist/cli/cli.js +54 -0
  7. package/dist/cli/cli.js.map +1 -0
  8. package/dist/cli/cliCommand.d.ts +17 -0
  9. package/dist/cli/cliCommand.d.ts.map +1 -0
  10. package/dist/cli/cliCommand.js +36 -0
  11. package/dist/cli/cliCommand.js.map +1 -0
  12. package/dist/cli/cliCommandComposite.d.ts +14 -0
  13. package/dist/cli/cliCommandComposite.d.ts.map +1 -0
  14. package/dist/cli/cliCommandComposite.js +34 -0
  15. package/dist/cli/cliCommandComposite.js.map +1 -0
  16. package/dist/cli.js +21 -35
  17. package/dist/cli.js.map +1 -1
  18. package/dist/commands/flatten.d.ts +56 -0
  19. package/dist/commands/flatten.d.ts.map +1 -0
  20. package/dist/commands/flatten.js +103 -0
  21. package/dist/commands/flatten.js.map +1 -0
  22. package/dist/commands/generate.d.ts +114 -0
  23. package/dist/commands/generate.d.ts.map +1 -0
  24. package/dist/commands/generate.js +151 -0
  25. package/dist/commands/generate.js.map +1 -0
  26. package/dist/commands/generateV1.d.ts +174 -0
  27. package/dist/commands/generateV1.d.ts.map +1 -0
  28. package/dist/commands/generateV1.js +178 -0
  29. package/dist/commands/generateV1.js.map +1 -0
  30. package/dist/commands/generateV2.d.ts +184 -0
  31. package/dist/commands/generateV2.d.ts.map +1 -0
  32. package/dist/commands/generateV2.js +204 -0
  33. package/dist/commands/generateV2.js.map +1 -0
  34. package/dist/commands/get.d.ts +35 -0
  35. package/dist/commands/get.d.ts.map +1 -0
  36. package/dist/commands/get.js +81 -0
  37. package/dist/commands/get.js.map +1 -0
  38. package/dist/commands/labels.d.ts +46 -0
  39. package/dist/commands/labels.d.ts.map +1 -0
  40. package/dist/commands/labels.js +43 -0
  41. package/dist/commands/labels.js.map +1 -0
  42. package/dist/commands/relocate.d.ts +68 -0
  43. package/dist/commands/relocate.d.ts.map +1 -0
  44. package/dist/commands/relocate.js +75 -0
  45. package/dist/commands/relocate.js.map +1 -0
  46. package/dist/commands/resize.d.ts +66 -0
  47. package/dist/commands/resize.d.ts.map +1 -0
  48. package/dist/commands/resize.js +118 -0
  49. package/dist/commands/resize.js.map +1 -0
  50. package/dist/{generate/distinct/dummy.d.ts.map → dummy.d.ts.map} +1 -1
  51. package/dist/dummy.js.map +1 -0
  52. package/dist/index.d.ts +1 -1
  53. package/dist/index.d.ts.map +1 -1
  54. package/dist/index.js +1 -1
  55. package/dist/index.js.map +1 -1
  56. package/dist/tests/data/flat_resized/labels.json +3 -0
  57. package/dist/tests/data/flat_resized/relocated_data.json +364 -0
  58. package/dist/tests/mocked.test.d.ts +2 -0
  59. package/dist/tests/mocked.test.d.ts.map +1 -0
  60. package/dist/tests/mocked.test.js +271 -0
  61. package/dist/tests/mocked.test.js.map +1 -0
  62. package/dist/tests/utils.d.ts +35 -0
  63. package/dist/tests/utils.d.ts.map +1 -0
  64. package/dist/tests/utils.js +120 -0
  65. package/dist/tests/utils.js.map +1 -0
  66. package/dist/utils/input.d.ts +24 -0
  67. package/dist/utils/input.d.ts.map +1 -0
  68. package/dist/utils/input.js +34 -0
  69. package/dist/utils/input.js.map +1 -0
  70. package/dist/utils/inputOutput.d.ts +20 -0
  71. package/dist/utils/inputOutput.d.ts.map +1 -0
  72. package/dist/utils/inputOutput.js +10 -0
  73. package/dist/utils/inputOutput.js.map +1 -0
  74. package/dist/utils/output.d.ts +37 -0
  75. package/dist/utils/output.d.ts.map +1 -0
  76. package/dist/utils/output.js +63 -0
  77. package/dist/utils/output.js.map +1 -0
  78. package/package.json +5 -6
  79. package/dist/cjs/cli.cjs +0 -34
  80. package/dist/cjs/flatten/args.cjs +0 -9
  81. package/dist/cjs/flatten/cli.cjs +0 -28
  82. package/dist/cjs/flatten/flatten.cjs +0 -55
  83. package/dist/cjs/generate/args.cjs +0 -16
  84. package/dist/cjs/generate/cli.cjs +0 -50
  85. package/dist/cjs/generate/distinct/args.cjs +0 -11
  86. package/dist/cjs/generate/distinct/cli.cjs +0 -28
  87. package/dist/cjs/generate/distinct/generate.cjs +0 -174
  88. package/dist/cjs/generate/union/args.cjs +0 -12
  89. package/dist/cjs/generate/union/cli.cjs +0 -31
  90. package/dist/cjs/generate/union/generate.cjs +0 -151
  91. package/dist/cjs/generate/util.cjs +0 -21
  92. package/dist/cjs/get/args.cjs +0 -7
  93. package/dist/cjs/get/cli.cjs +0 -20
  94. package/dist/cjs/get/get.cjs +0 -53
  95. package/dist/cjs/labels/args.cjs +0 -7
  96. package/dist/cjs/labels/cli.cjs +0 -20
  97. package/dist/cjs/labels/labels.cjs +0 -21
  98. package/dist/cjs/relocate/args.cjs +0 -9
  99. package/dist/cjs/relocate/cli.cjs +0 -27
  100. package/dist/cjs/relocate/relocate.cjs +0 -36
  101. package/dist/cjs/scale/args.cjs +0 -11
  102. package/dist/cjs/scale/cli.cjs +0 -34
  103. package/dist/cjs/scale/scale.cjs +0 -55
  104. package/dist/flatten/args.d.ts +0 -16
  105. package/dist/flatten/args.d.ts.map +0 -1
  106. package/dist/flatten/args.js +0 -8
  107. package/dist/flatten/args.js.map +0 -1
  108. package/dist/flatten/cli.d.ts +0 -18
  109. package/dist/flatten/cli.d.ts.map +0 -1
  110. package/dist/flatten/cli.js +0 -30
  111. package/dist/flatten/cli.js.map +0 -1
  112. package/dist/flatten/flatten.d.ts +0 -5
  113. package/dist/flatten/flatten.d.ts.map +0 -1
  114. package/dist/flatten/flatten.js +0 -65
  115. package/dist/flatten/flatten.js.map +0 -1
  116. package/dist/flatten/index.d.ts +0 -4
  117. package/dist/flatten/index.d.ts.map +0 -1
  118. package/dist/flatten/index.js +0 -4
  119. package/dist/flatten/index.js.map +0 -1
  120. package/dist/generate/args.d.ts +0 -37
  121. package/dist/generate/args.d.ts.map +0 -1
  122. package/dist/generate/args.js +0 -15
  123. package/dist/generate/args.js.map +0 -1
  124. package/dist/generate/cli.d.ts +0 -32
  125. package/dist/generate/cli.d.ts.map +0 -1
  126. package/dist/generate/cli.js +0 -62
  127. package/dist/generate/cli.js.map +0 -1
  128. package/dist/generate/distinct/args.d.ts +0 -49
  129. package/dist/generate/distinct/args.d.ts.map +0 -1
  130. package/dist/generate/distinct/args.js +0 -10
  131. package/dist/generate/distinct/args.js.map +0 -1
  132. package/dist/generate/distinct/cli.d.ts +0 -20
  133. package/dist/generate/distinct/cli.d.ts.map +0 -1
  134. package/dist/generate/distinct/cli.js +0 -31
  135. package/dist/generate/distinct/cli.js.map +0 -1
  136. package/dist/generate/distinct/dummy.js.map +0 -1
  137. package/dist/generate/distinct/generate.d.ts +0 -5
  138. package/dist/generate/distinct/generate.d.ts.map +0 -1
  139. package/dist/generate/distinct/generate.js +0 -186
  140. package/dist/generate/distinct/generate.js.map +0 -1
  141. package/dist/generate/distinct/index.d.ts +0 -4
  142. package/dist/generate/distinct/index.d.ts.map +0 -1
  143. package/dist/generate/distinct/index.js +0 -4
  144. package/dist/generate/distinct/index.js.map +0 -1
  145. package/dist/generate/index.d.ts +0 -5
  146. package/dist/generate/index.d.ts.map +0 -1
  147. package/dist/generate/index.js +0 -5
  148. package/dist/generate/index.js.map +0 -1
  149. package/dist/generate/union/args.d.ts +0 -52
  150. package/dist/generate/union/args.d.ts.map +0 -1
  151. package/dist/generate/union/args.js +0 -11
  152. package/dist/generate/union/args.js.map +0 -1
  153. package/dist/generate/union/cli.d.ts +0 -22
  154. package/dist/generate/union/cli.d.ts.map +0 -1
  155. package/dist/generate/union/cli.js +0 -35
  156. package/dist/generate/union/cli.js.map +0 -1
  157. package/dist/generate/union/generate.d.ts +0 -5
  158. package/dist/generate/union/generate.d.ts.map +0 -1
  159. package/dist/generate/union/generate.js +0 -168
  160. package/dist/generate/union/generate.js.map +0 -1
  161. package/dist/generate/union/index.d.ts +0 -4
  162. package/dist/generate/union/index.d.ts.map +0 -1
  163. package/dist/generate/union/index.js +0 -4
  164. package/dist/generate/union/index.js.map +0 -1
  165. package/dist/generate/util.d.ts +0 -6
  166. package/dist/generate/util.d.ts.map +0 -1
  167. package/dist/generate/util.js +0 -20
  168. package/dist/generate/util.js.map +0 -1
  169. package/dist/get/args.d.ts +0 -10
  170. package/dist/get/args.d.ts.map +0 -1
  171. package/dist/get/args.js +0 -6
  172. package/dist/get/args.js.map +0 -1
  173. package/dist/get/cli.d.ts +0 -14
  174. package/dist/get/cli.d.ts.map +0 -1
  175. package/dist/get/cli.js +0 -19
  176. package/dist/get/cli.js.map +0 -1
  177. package/dist/get/get.d.ts +0 -5
  178. package/dist/get/get.d.ts.map +0 -1
  179. package/dist/get/get.js +0 -60
  180. package/dist/get/get.js.map +0 -1
  181. package/dist/get/index.d.ts +0 -4
  182. package/dist/get/index.d.ts.map +0 -1
  183. package/dist/get/index.js +0 -4
  184. package/dist/get/index.js.map +0 -1
  185. package/dist/labels/args.d.ts +0 -10
  186. package/dist/labels/args.d.ts.map +0 -1
  187. package/dist/labels/args.js +0 -6
  188. package/dist/labels/args.js.map +0 -1
  189. package/dist/labels/cli.d.ts +0 -14
  190. package/dist/labels/cli.d.ts.map +0 -1
  191. package/dist/labels/cli.js +0 -19
  192. package/dist/labels/cli.js.map +0 -1
  193. package/dist/labels/index.d.ts +0 -4
  194. package/dist/labels/index.d.ts.map +0 -1
  195. package/dist/labels/index.js +0 -4
  196. package/dist/labels/index.js.map +0 -1
  197. package/dist/labels/labels.d.ts +0 -5
  198. package/dist/labels/labels.d.ts.map +0 -1
  199. package/dist/labels/labels.js +0 -22
  200. package/dist/labels/labels.js.map +0 -1
  201. package/dist/relocate/args.d.ts +0 -16
  202. package/dist/relocate/args.d.ts.map +0 -1
  203. package/dist/relocate/args.js +0 -8
  204. package/dist/relocate/args.js.map +0 -1
  205. package/dist/relocate/cli.d.ts +0 -18
  206. package/dist/relocate/cli.d.ts.map +0 -1
  207. package/dist/relocate/cli.js +0 -29
  208. package/dist/relocate/cli.js.map +0 -1
  209. package/dist/relocate/relocate.d.ts +0 -5
  210. package/dist/relocate/relocate.d.ts.map +0 -1
  211. package/dist/relocate/relocate.js +0 -40
  212. package/dist/relocate/relocate.js.map +0 -1
  213. package/dist/scale/args.d.ts +0 -22
  214. package/dist/scale/args.d.ts.map +0 -1
  215. package/dist/scale/args.js +0 -10
  216. package/dist/scale/args.js.map +0 -1
  217. package/dist/scale/cli.d.ts +0 -22
  218. package/dist/scale/cli.d.ts.map +0 -1
  219. package/dist/scale/cli.js +0 -38
  220. package/dist/scale/cli.js.map +0 -1
  221. package/dist/scale/index.d.ts +0 -4
  222. package/dist/scale/index.d.ts.map +0 -1
  223. package/dist/scale/index.js +0 -4
  224. package/dist/scale/index.js.map +0 -1
  225. package/dist/scale/scale.d.ts +0 -5
  226. package/dist/scale/scale.d.ts.map +0 -1
  227. package/dist/scale/scale.js +0 -67
  228. package/dist/scale/scale.js.map +0 -1
  229. /package/dist/{generate/distinct/dummy.d.ts → dummy.d.ts} +0 -0
  230. /package/dist/{generate/distinct/dummy.js → dummy.js} +0 -0
@@ -1,11 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const args = require("../args.cjs");
4
- const zod = require("zod");
5
- const argsSchema = args.argsSchema.extend({
6
- solved: zod.z.number().optional(),
7
- unsolved: zod.z.number().optional(),
8
- minCorrect: zod.z.number().optional(),
9
- maxCorrect: zod.z.number().optional()
10
- });
11
- exports.argsSchema = argsSchema;
@@ -1,28 +0,0 @@
1
- "use strict";
2
- const args = require("./args.cjs");
3
- const generate = require("./generate.cjs");
4
- const generateDistinct = (cmdArgs) => {
5
- return {
6
- command: "distinct",
7
- describe: "Generate distinct captchas producing captcha challenges comprising 2 rounds, one labelled and one unlabelled",
8
- builder: (yargs) => {
9
- return yargs.option("solved", {
10
- type: "number",
11
- description: "Number of captchas to generate that are solved"
12
- }).option("unsolved", {
13
- type: "number",
14
- description: "Number of captchas to generate that are unsolved"
15
- }).option("min-correct", {
16
- type: "number",
17
- description: "Minimum number of target images in each captcha"
18
- }).option("max-correct", {
19
- type: "number",
20
- description: "Maximum number of target images in each captcha"
21
- });
22
- },
23
- handler: async (argv) => {
24
- await generate(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
25
- }
26
- };
27
- };
28
- module.exports = generateDistinct;
@@ -1,174 +0,0 @@
1
- "use strict";
2
- const types = require("@prosopo/types");
3
- const common = require("@prosopo/common");
4
- const util = require("@prosopo/util");
5
- const utilCrypto = require("@polkadot/util-crypto");
6
- const util$1 = require("../util.cjs");
7
- const bcrypt = require("bcrypt");
8
- const cliProgress = require("cli-progress");
9
- const fs = require("fs");
10
- const generate = async (args, logger) => {
11
- logger = logger || common.getLoggerDefault();
12
- logger.debug(args, "generating...");
13
- const outFile = args.out;
14
- const overwrite = args.overwrite || false;
15
- if (!overwrite && fs.existsSync(outFile)) {
16
- throw new common.ProsopoEnvError(new Error(`output file already exists: ${outFile}`), "FS.FILE_ALREADY_EXISTS");
17
- }
18
- const labelledMapFile = args.labelled;
19
- if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
20
- throw new common.ProsopoEnvError(
21
- new Error(`labelled map file does not exist: ${labelledMapFile}`),
22
- "FS.FILE_NOT_FOUND"
23
- );
24
- }
25
- const unlabelledMapFile = args.unlabelled;
26
- if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
27
- throw new common.ProsopoEnvError(
28
- new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`),
29
- "FS.FILE_NOT_FOUND"
30
- );
31
- }
32
- const labelsFile = args.labels;
33
- const seed = args.seed || 0;
34
- const size = args.size || 9;
35
- const minCorrect = args.minCorrect || 1;
36
- const maxCorrect = args.maxCorrect || size - 1;
37
- const solved = args.solved || 0;
38
- const unsolved = args.unsolved || 0;
39
- const saltRounds = 10;
40
- const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false;
41
- const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
42
- util.setSeedGlobal(seed);
43
- const _ = util.lodash();
44
- const labelled = labelledMapFile ? types.LabelledDataSchema.parse(JSON.parse(fs.readFileSync(labelledMapFile, "utf8"))).items : [];
45
- const unlabelled = unlabelledMapFile ? types.DataSchema.parse(JSON.parse(fs.readFileSync(unlabelledMapFile, "utf8"))).items : [];
46
- util$1.checkDuplicates(labelled, unlabelled, {
47
- allowDuplicatesLabelled,
48
- allowDuplicatesUnlabelled
49
- });
50
- const labelToImages = {};
51
- for (const entry of labelled) {
52
- const arr = labelToImages[entry.label] || [];
53
- arr.push(entry);
54
- labelToImages[entry.label] = arr;
55
- }
56
- const targets = Object.keys(labelToImages);
57
- const labels = [];
58
- if (labelsFile && fs.existsSync(labelsFile)) {
59
- labels.push(...[...types.LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(labelsFile, "utf8"))).labels]);
60
- } else {
61
- labels.push(...[...targets]);
62
- }
63
- const solvedCaptchas = [];
64
- const barSolved = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
65
- logger.info(`Generating ${solved} solved captchas...`);
66
- barSolved.start(solved, 0);
67
- for (let i = 0; i < solved; i++) {
68
- barSolved.update(i + 1);
69
- if (targets.length <= 1) {
70
- throw new common.ProsopoEnvError(
71
- new Error(`not enough different labels in labelled data: ${labelledMapFile}`),
72
- "DATASET.NOT_ENOUGH_LABELS"
73
- );
74
- }
75
- const target = util.at(targets, i % targets.length);
76
- const notTargets = targets.filter((t) => t !== target);
77
- const nCorrect = _.random(minCorrect, maxCorrect);
78
- const nIncorrect = size - nCorrect;
79
- const targetItems = util.get(labelToImages, target);
80
- const notTargetItems = notTargets.map((notTarget) => util.get(labelToImages, notTarget)).flat();
81
- if (targetItems.length < nCorrect) {
82
- throw new common.ProsopoEnvError(
83
- new Error(`not enough images for target (${target})`),
84
- "DATASET.NOT_ENOUGH_IMAGES"
85
- );
86
- }
87
- if (notTargetItems.length < nIncorrect) {
88
- throw new common.ProsopoEnvError(
89
- new Error(`not enough non-matching images for target (${target})`),
90
- "DATASET.NOT_ENOUGH_IMAGES"
91
- );
92
- }
93
- const correctItems = _.sampleSize(targetItems, nCorrect);
94
- const incorrectItems = _.sampleSize(notTargetItems, nIncorrect);
95
- let items = [...correctItems, ...incorrectItems];
96
- let indices = [...Array(items.length).keys()];
97
- indices = _.shuffle(indices);
98
- items = indices.map((i2) => util.at(items, i2));
99
- items = items.map((item) => {
100
- return {
101
- data: item.data,
102
- hash: item.hash,
103
- type: item.type
104
- };
105
- });
106
- const solution = indices.map((index, i2) => {
107
- return {
108
- pre: index,
109
- // the index of the item in the items array before shuffle
110
- post: i2
111
- // the index of the item in the shuffled array
112
- };
113
- }).filter((item) => item.pre < correctItems.length).map((item) => {
114
- return item.post;
115
- });
116
- const salt = utilCrypto.blake2AsHex(bcrypt.genSaltSync(saltRounds));
117
- const captcha = {
118
- salt,
119
- target,
120
- items,
121
- solution
122
- };
123
- solvedCaptchas.push(captcha);
124
- }
125
- barSolved.stop();
126
- logger.info(`Generating ${unsolved} unsolved captchas...`);
127
- const barUnsolved = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
128
- barUnsolved.start(unsolved, 0);
129
- const unsolvedCaptchas = [];
130
- for (let i = 0; i < unsolved; i++) {
131
- barUnsolved.update(i + 1);
132
- if (unlabelled.length <= size) {
133
- throw new common.ProsopoEnvError(
134
- new Error(`unlabelled map file does not contain enough data: ${unlabelledMapFile}`),
135
- "DATASET.NOT_ENOUGH_IMAGES"
136
- );
137
- }
138
- if (labels.length <= 0) {
139
- throw new common.ProsopoEnvError(
140
- new Error(`no labels found for unlabelled data: ${labelsFile}`),
141
- "DATASET.NOT_ENOUGH_LABELS"
142
- );
143
- }
144
- const index = _.random(0, labels.length - 1);
145
- const target = util.at(labels, index);
146
- const itemSet = _.sampleSize(unlabelled, size);
147
- let items = [...itemSet];
148
- let indices = [...Array(items.length).keys()];
149
- indices = _.shuffle(indices);
150
- items = indices.map((i2) => util.at(items, i2));
151
- items = items.map((item) => {
152
- return {
153
- data: item.data,
154
- hash: item.hash,
155
- type: item.type
156
- };
157
- });
158
- const salt = utilCrypto.blake2AsHex(bcrypt.genSaltSync(saltRounds));
159
- const captcha = {
160
- salt,
161
- target,
162
- items
163
- };
164
- unsolvedCaptchas.push(captcha);
165
- }
166
- barUnsolved.stop();
167
- const output = {
168
- captchas: [...solvedCaptchas, ...unsolvedCaptchas],
169
- format: types.CaptchaTypes.SelectAll
170
- };
171
- types.CaptchasContainerSchema.parse(output);
172
- fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
173
- };
174
- module.exports = generate;
@@ -1,12 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const args = require("../args.cjs");
4
- const zod = require("zod");
5
- const argsSchema = args.argsSchema.extend({
6
- minCorrect: zod.z.number().optional(),
7
- minIncorrect: zod.z.number().optional(),
8
- minLabelled: zod.z.number().optional(),
9
- maxLabelled: zod.z.number().optional(),
10
- count: zod.z.number().optional()
11
- });
12
- exports.argsSchema = argsSchema;
@@ -1,31 +0,0 @@
1
- "use strict";
2
- const args = require("./args.cjs");
3
- const generate = require("./generate.cjs");
4
- const generateUnion = (cmdArgs) => {
5
- return {
6
- command: "union",
7
- describe: "Generate distinct captchas producing captcha challenges comprising one or more rounds, mixing labelled and unlabelled data into a single round",
8
- builder: (yargs) => {
9
- return yargs.option("count", {
10
- type: "number",
11
- description: "Number of captchas to generate"
12
- }).option("min-correct", {
13
- type: "number",
14
- description: "Minimum number of target images in each captcha"
15
- }).option("min-incorrect", {
16
- type: "number",
17
- description: "Minimum number of incorrect images in each captcha"
18
- }).option("min-labelled", {
19
- type: "number",
20
- description: "Minimum number of labelled images in each captcha"
21
- }).option("max-labelled", {
22
- type: "number",
23
- description: "Maximum number of labelled images in each captcha"
24
- });
25
- },
26
- handler: async (argv) => {
27
- await generate(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
28
- }
29
- };
30
- };
31
- module.exports = generateUnion;
@@ -1,151 +0,0 @@
1
- "use strict";
2
- const types = require("@prosopo/types");
3
- const common = require("@prosopo/common");
4
- const util = require("@prosopo/util");
5
- const utilCrypto = require("@polkadot/util-crypto");
6
- const util$1 = require("../util.cjs");
7
- const bcrypt = require("bcrypt");
8
- const fs = require("fs");
9
- const generate = async (args, logger) => {
10
- logger = logger || common.getLoggerDefault();
11
- logger.debug(args, "generating...");
12
- const outFile = args.out;
13
- const overwrite = args.overwrite || false;
14
- if (!overwrite && fs.existsSync(outFile)) {
15
- throw new common.ProsopoEnvError(new Error(`Output file already exists: ${outFile}`), "FS.FILE_ALREADY_EXISTS");
16
- }
17
- const labelledMapFile = args.labelled;
18
- if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
19
- throw new common.ProsopoEnvError(
20
- new Error(`Labelled map file does not exist: ${labelledMapFile}`),
21
- "FS.FILE_NOT_FOUND"
22
- );
23
- }
24
- const unlabelledMapFile = args.unlabelled;
25
- if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
26
- throw new common.ProsopoEnvError(
27
- new Error(`Unlabelled map file does not exist: ${unlabelledMapFile}`),
28
- "FS.FILE_NOT_FOUND"
29
- );
30
- }
31
- const labelsFile = args.labels;
32
- const seed = args.seed || 0;
33
- const size = args.size || 9;
34
- const minCorrect = args.minCorrect || 1;
35
- const saltRounds = 10;
36
- const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false;
37
- const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
38
- const minIncorrect = Math.max(args.minIncorrect || 1, 1);
39
- const minLabelled = minCorrect + minIncorrect;
40
- const maxLabelled = Math.min(args.maxLabelled || size, size);
41
- const count = args.count || 0;
42
- util.setSeedGlobal(seed);
43
- const _ = util.lodash();
44
- const labelled = labelledMapFile ? types.LabelledDataSchema.parse(JSON.parse(fs.readFileSync(labelledMapFile, "utf8"))).items : [];
45
- const unlabelled = unlabelledMapFile ? types.DataSchema.parse(JSON.parse(fs.readFileSync(unlabelledMapFile, "utf8"))).items : [];
46
- util$1.checkDuplicates(labelled, unlabelled, {
47
- allowDuplicatesLabelled,
48
- allowDuplicatesUnlabelled
49
- });
50
- const labelToImages = {};
51
- for (const entry of labelled) {
52
- const arr = labelToImages[entry.label] || [];
53
- arr.push(entry);
54
- labelToImages[entry.label] = arr;
55
- }
56
- const targets = Object.keys(labelToImages);
57
- const labels = [];
58
- if (labelsFile && fs.existsSync(labelsFile)) {
59
- labels.push(...[...types.LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(labelsFile, "utf8"))).labels]);
60
- } else {
61
- labels.push(...[...targets]);
62
- }
63
- const captchas = [];
64
- for (let i = 0; i < count; i++) {
65
- logger.info(`generating captcha ${i + 1} of ${count}`);
66
- if (targets.length <= 1) {
67
- throw new common.ProsopoEnvError(
68
- new Error(`not enough different labels in labelled data: ${labelledMapFile}`),
69
- "DATASET.NOT_ENOUGH_LABELS"
70
- );
71
- }
72
- const target = util.at(targets, i % targets.length);
73
- const notTargets = targets.filter((t) => t !== target);
74
- const nLabelled = _.random(minLabelled, maxLabelled);
75
- const maxCorrect = nLabelled - minCorrect;
76
- const nCorrect = _.random(minCorrect, maxCorrect);
77
- const nIncorrect = nLabelled - nCorrect;
78
- const nUnlabelled = size - nLabelled;
79
- const targetItems = util.get(labelToImages, target);
80
- const notTargetItems = notTargets.map((notTarget) => util.get(labelToImages, notTarget)).flat();
81
- if (nUnlabelled > unlabelled.length) {
82
- throw new common.ProsopoEnvError(new Error(`not enough unlabelled data`), "DATASET.NOT_ENOUGH_IMAGES");
83
- }
84
- if (nCorrect > targetItems.length) {
85
- throw new common.ProsopoEnvError(
86
- new Error(`not enough images for target (${target})`),
87
- "DATASET.NOT_ENOUGH_IMAGES"
88
- );
89
- }
90
- if (nIncorrect > notTargetItems.length) {
91
- throw new common.ProsopoEnvError(
92
- new Error(`not enough non-matching images for target (${target})`),
93
- "DATASET.NOT_ENOUGH_IMAGES"
94
- );
95
- }
96
- const correctItems = _.sampleSize(targetItems, nCorrect);
97
- const incorrectItems = _.sampleSize(notTargetItems, nIncorrect);
98
- const unlabelledItems = /* @__PURE__ */ new Set();
99
- while (unlabelledItems.size < size - nLabelled) {
100
- const image = util.at(unlabelled, _.random(0, unlabelled.length - 1));
101
- unlabelledItems.add(image);
102
- }
103
- let items = [...correctItems, ...incorrectItems, ...unlabelledItems];
104
- let indices = [...Array(items.length).keys()];
105
- indices = _.shuffle(indices);
106
- items = indices.map((i2) => util.at(items, i2));
107
- items = items.map((item) => {
108
- return {
109
- data: item.data,
110
- hash: item.hash,
111
- type: item.type
112
- };
113
- });
114
- const solution = indices.map((index, i2) => {
115
- return {
116
- pre: index,
117
- // the index of the item in the items array before shuffle
118
- post: i2
119
- // the index of the item in the shuffled array
120
- };
121
- }).filter((item) => item.pre < correctItems.length).map((item) => {
122
- return item.post;
123
- });
124
- const unlabelledIndices = indices.map((index, i2) => {
125
- return {
126
- pre: index,
127
- // the index of the item in the items array before shuffle
128
- post: i2
129
- // the index of the item in the shuffled array
130
- };
131
- }).filter((item) => item.pre >= correctItems.length + incorrectItems.length).map((item) => {
132
- return item.post;
133
- });
134
- const salt = utilCrypto.blake2AsHex(bcrypt.genSaltSync(saltRounds));
135
- const captcha = {
136
- salt,
137
- target,
138
- items,
139
- solution,
140
- unlabelled: unlabelledIndices
141
- };
142
- captchas.push(captcha);
143
- }
144
- const output = {
145
- captchas,
146
- format: types.CaptchaTypes.SelectAll
147
- };
148
- types.CaptchasContainerSchema.parse(output);
149
- fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
150
- };
151
- module.exports = generate;
@@ -1,21 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const checkDuplicates = (labelled, unlabelled, options) => {
4
- const all = /* @__PURE__ */ new Set();
5
- if (!options.allowDuplicatesLabelled) {
6
- for (const entry of labelled) {
7
- if (all.has(entry.data)) {
8
- throw new Error(`Duplicate data entry in labelled data: ${JSON.stringify(entry)}`);
9
- }
10
- all.add(entry.data);
11
- }
12
- }
13
- if (!options.allowDuplicatesUnlabelled) {
14
- for (const entry of unlabelled) {
15
- if (all.has(entry.data)) {
16
- throw new Error(`Duplicate data entry in unlabelled data: ${JSON.stringify(entry)}`);
17
- }
18
- }
19
- }
20
- };
21
- exports.checkDuplicates = checkDuplicates;
@@ -1,7 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const zod = require("zod");
4
- const argsSchema = zod.z.object({
5
- data: zod.z.string()
6
- });
7
- exports.argsSchema = argsSchema;
@@ -1,20 +0,0 @@
1
- "use strict";
2
- const args = require("./args.cjs");
3
- const get$1 = require("./get.cjs");
4
- const get = (cmdArgs) => {
5
- return {
6
- command: "get",
7
- describe: "Test a GET request at image URLs",
8
- builder: (yargs) => {
9
- return yargs.option("data", {
10
- type: "string",
11
- demand: true,
12
- description: 'JSON file containing urls under a "data" key'
13
- });
14
- },
15
- handler: async (argv) => {
16
- await get$1(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
17
- }
18
- };
19
- };
20
- module.exports = get;
@@ -1,53 +0,0 @@
1
- "use strict";
2
- const common = require("@prosopo/common");
3
- const util = require("@prosopo/util");
4
- const zod = require("zod");
5
- const fetch = require("node-fetch");
6
- const fs = require("fs");
7
- const get = async (args, loggerOpt) => {
8
- const logger = loggerOpt || common.getLoggerDefault();
9
- logger.debug(args, "getting...");
10
- const traverse = async (data2) => {
11
- if (data2 instanceof Array) {
12
- for (let i = 0; i < data2.length; i++) {
13
- data2[i] = await traverse(data2[i]);
14
- }
15
- } else if (data2 instanceof Object) {
16
- for (const key of Object.keys(data2)) {
17
- if (key == "data") {
18
- const value = util.get(data2, key);
19
- const url = zod.z.string().parse(value);
20
- if (url.startsWith("http")) {
21
- try {
22
- const response = await fetch(url);
23
- if (!response.ok) {
24
- logger.error(`GET ${url} ${response.status} ${response.statusText}`);
25
- } else {
26
- logger.log(`GET ${url} OK`);
27
- }
28
- } catch (err) {
29
- logger.error(err);
30
- }
31
- } else {
32
- try {
33
- fs.readFileSync(url);
34
- logger.log(`GET ${url} OK`);
35
- } catch (err) {
36
- logger.error(`GET ${url} ${err}`);
37
- }
38
- }
39
- } else {
40
- await traverse(util.get(data2, key));
41
- }
42
- }
43
- }
44
- return data2;
45
- };
46
- const file = args.data;
47
- if (!fs.existsSync(file)) {
48
- throw new common.ProsopoEnvError(new Error(`file does not exist: ${file}`), "FS.FILE_NOT_FOUND");
49
- }
50
- const data = JSON.parse(fs.readFileSync(file, "utf8"));
51
- await traverse(data);
52
- };
53
- module.exports = get;
@@ -1,7 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const zod = require("zod");
4
- const argsSchema = zod.z.object({
5
- data: zod.z.string()
6
- });
7
- exports.argsSchema = argsSchema;
@@ -1,20 +0,0 @@
1
- "use strict";
2
- const args = require("./args.cjs");
3
- const labels$1 = require("./labels.cjs");
4
- const labels = (cmdArgs) => {
5
- return {
6
- command: "labels",
7
- describe: "get all labels from some data",
8
- builder: (yargs) => {
9
- return yargs.option("data", {
10
- type: "string",
11
- demand: true,
12
- description: "JSON file containing data"
13
- });
14
- },
15
- handler: async (argv) => {
16
- await labels$1(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
17
- }
18
- };
19
- };
20
- module.exports = labels;
@@ -1,21 +0,0 @@
1
- "use strict";
2
- const types = require("@prosopo/types");
3
- const common = require("@prosopo/common");
4
- const fs = require("fs");
5
- const labels = async (args, logger) => {
6
- logger = logger || common.getLoggerDefault();
7
- logger.debug(args, "reading labels...");
8
- const file = args.data;
9
- if (!fs.existsSync(file)) {
10
- throw new common.ProsopoEnvError(new Error(`file does not exist: ${file}`), "FS.FILE_NOT_FOUND");
11
- }
12
- const labelled = file ? types.LabelledDataSchema.parse(JSON.parse(fs.readFileSync(file, "utf8"))).items : [];
13
- const labels2 = /* @__PURE__ */ new Set();
14
- for (const item of labelled) {
15
- labels2.add(item.label);
16
- }
17
- const labelArray = Array.from(labels2);
18
- labelArray.sort();
19
- logger.log(JSON.stringify({ labels: labelArray }, null, 4));
20
- };
21
- module.exports = labels;
@@ -1,9 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const zod = require("zod");
4
- const argsSchema = zod.z.object({
5
- data: zod.z.string(),
6
- from: zod.z.string(),
7
- to: zod.z.string()
8
- });
9
- exports.argsSchema = argsSchema;
@@ -1,27 +0,0 @@
1
- "use strict";
2
- const args = require("./args.cjs");
3
- const relocate$1 = require("./relocate.cjs");
4
- const relocate = (cmdArgs) => {
5
- return {
6
- command: "relocate",
7
- describe: 'Relocate a dataset by replacing the old urls with new ones. E.g. "example.com/1.jpg" to "newwebsite.com/1.jpg"',
8
- builder: (yargs) => {
9
- return yargs.option("from", {
10
- type: "string",
11
- demand: true,
12
- description: "The old url to replace"
13
- }).option("to", {
14
- type: "string",
15
- demand: true,
16
- description: "The new url to replace the old one with"
17
- }).option("data", {
18
- type: "string",
19
- description: "Path to the images JSON containing the urls of images to replace"
20
- });
21
- },
22
- handler: async (argv) => {
23
- await relocate$1(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
24
- }
25
- };
26
- };
27
- module.exports = relocate;
@@ -1,36 +0,0 @@
1
- "use strict";
2
- const common = require("@prosopo/common");
3
- const util = require("@prosopo/util");
4
- const fs = require("fs");
5
- const relocate = async (args, logger) => {
6
- logger = logger || common.getLoggerDefault();
7
- logger.debug(args, "relocating...");
8
- const replace = (data2, from, to) => {
9
- if (Array.isArray(data2)) {
10
- for (let i = 0; i < data2.length; i++) {
11
- data2[i] = replace(data2[i], from, to);
12
- }
13
- } else if (typeof data2 === "object") {
14
- const obj = data2;
15
- for (const key of Object.keys(obj)) {
16
- if (key === "data") {
17
- const value = util.get(obj, key);
18
- if (value instanceof String) {
19
- if (value.startsWith(from)) {
20
- obj[key] = to + value.slice(from.length);
21
- }
22
- }
23
- } else {
24
- obj[key] = replace(obj[key], from, to);
25
- }
26
- }
27
- }
28
- return data2;
29
- };
30
- const file = args.data;
31
- logger.log(`relocating data in ${file} from ${args.from} to ${args.to}`);
32
- let data = JSON.parse(fs.readFileSync(file, "utf8"));
33
- data = replace(data, args.from, args.to);
34
- fs.writeFileSync(file, JSON.stringify(data, null, 4));
35
- };
36
- module.exports = relocate;
@@ -1,11 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
- const zod = require("zod");
4
- const argsSchema = zod.z.object({
5
- data: zod.z.string(),
6
- out: zod.z.string(),
7
- overwrite: zod.z.boolean().optional(),
8
- size: zod.z.number(),
9
- square: zod.z.boolean().optional()
10
- });
11
- exports.argsSchema = argsSchema;