@prosopo/datasets-fs 3.0.136 → 3.0.138

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
 
2
- > @prosopo/datasets-fs@3.0.136 build:cjs
2
+ > @prosopo/datasets-fs@3.0.138 build:cjs
3
3
  > NODE_ENV=${NODE_ENV:-development}; vite build --config vite.cjs.config.ts --mode $NODE_ENV
4
4
 
5
5
  ViteCommonJSConfig: .
@@ -7,23 +7,23 @@ ViteCommonJSConfig: .
7
7
  tsConfigPaths: [
8
8
  '/home/runner/work/captcha/captcha/packages/common/tsconfig.json',
9
9
  '/home/runner/work/captcha/captcha/packages/locale/tsconfig.json',
10
- '/home/runner/work/captcha/captcha/packages/logger/tsconfig.json',
11
10
  '/home/runner/work/captcha/captcha/packages/util/tsconfig.json',
11
+ '/home/runner/work/captcha/captcha/packages/logger/tsconfig.json',
12
12
  '/home/runner/work/captcha/captcha/packages/types/tsconfig.json',
13
13
  '/home/runner/work/captcha/captcha/packages/util-crypto/tsconfig.json'
14
14
  ]
15
15
  }
16
16
  {
17
17
  externals: [
18
+ The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the resolve.tsconfigPaths option. You can remove the plugin and set resolve.tsconfigPaths: true in your Vite config instead.
18
19
  '@prosopo/common',
19
20
  '@prosopo/locale',
20
- '@prosopo/logger',
21
21
  '@prosopo/util',
22
+ '@prosopo/logger',
22
23
  '@prosopo/types',
23
24
  '@prosopo/util-crypto'
24
25
  ]
25
26
  }
26
- The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the resolve.tsconfigPaths option. You can remove the plugin and set resolve.tsconfigPaths: true in your Vite config instead.
27
27
  vite v8.1.5 building ssr environment for production...
28
28
  Bundle build started
29
29
  
@@ -34,7 +34,7 @@ dist/cjs/index.cjs 0.15 kB │ gzip: 0.14 kB
34
34
  dist/cjs/_virtual/_rolldown/runtime.cjs 1.00 kB │ gzip: 0.50 kB
35
35
  dist/cjs/cli/cli.cjs 1.74 kB │ gzip: 0.81 kB
36
36
 
37
- ✓ built in 62ms
37
+ ✓ built in 70ms
38
38
  [copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat/data.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat/data.json
39
39
  [copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat_resized/captchas_v1.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat_resized/captchas_v1.json
40
40
  [copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat_resized/captchas_v2.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat_resized/captchas_v2.json
@@ -1,11 +1,11 @@
1
1
 
2
- > @prosopo/datasets-fs@3.0.136 build:tsc
2
+ > @prosopo/datasets-fs@3.0.138 build:tsc
3
3
  > tsc --build --verbose
4
4
 
5
- 11:04:10 AM - Projects in this build:
5
+ 9:05:45 AM - Projects in this build:
6
6
  * ../../dev/config/tsconfig.json
7
- * ../locale/tsconfig.json
8
7
  * ../util/tsconfig.json
8
+ * ../locale/tsconfig.json
9
9
  * ../logger/tsconfig.json
10
10
  * ../util-crypto/tsconfig.json
11
11
  * ../types/tsconfig.json
@@ -13,23 +13,23 @@
13
13
  * ../../dev/workspace/tsconfig.json
14
14
  * tsconfig.json
15
15
 
16
- 11:04:10 AM - Project '../../dev/config/tsconfig.json' is up to date because newest input '../../dev/config/src/vite/NodejsPolarsDirnamePlugin.ts' is older than output '../../dev/config/tsconfig.tsbuildinfo'
16
+ 9:05:45 AM - Project '../../dev/config/tsconfig.json' is up to date because newest input '../../dev/config/src/webpack/webpack.config.ts' is older than output '../../dev/config/tsconfig.tsbuildinfo'
17
17
 
18
- 11:04:10 AM - Project '../locale/tsconfig.json' is up to date because newest input '../locale/src/process.ts' is older than output '../locale/tsconfig.tsbuildinfo'
18
+ 9:05:45 AM - Project '../util/tsconfig.json' is up to date because newest input '../util/src/url.ts' is older than output '../util/tsconfig.tsbuildinfo'
19
19
 
20
- 11:04:10 AM - Project '../util/tsconfig.json' is up to date because newest input '../util/src/ofLen.ts' is older than output '../util/tsconfig.tsbuildinfo'
20
+ 9:05:45 AM - Project '../locale/tsconfig.json' is up to date because newest input '../locale/src/translationKey.ts' is older than output '../locale/tsconfig.tsbuildinfo'
21
21
 
22
- 11:04:10 AM - Project '../logger/tsconfig.json' is up to date because newest input '../logger/src/index.ts' is older than output '../logger/tsconfig.tsbuildinfo'
22
+ 9:05:45 AM - Project '../logger/tsconfig.json' is up to date because newest input '../logger/src/index.ts' is older than output '../logger/tsconfig.tsbuildinfo'
23
23
 
24
- 11:04:10 AM - Project '../util-crypto/tsconfig.json' is up to date because newest input '../util-crypto/src/types.ts' is older than output '../util-crypto/tsconfig.tsbuildinfo'
24
+ 9:05:45 AM - Project '../util-crypto/tsconfig.json' is up to date because newest input '../util-crypto/src/types.ts' is older than output '../util-crypto/tsconfig.tsbuildinfo'
25
25
 
26
- 11:04:10 AM - Project '../types/tsconfig.json' is up to date because newest input '../types/src/types.test-d.ts' is older than output '../types/tsconfig.tsbuildinfo'
26
+ 9:05:45 AM - Project '../types/tsconfig.json' is up to date because newest input '../types/src/types.test-d.ts' is older than output '../types/tsconfig.tsbuildinfo'
27
27
 
28
- 11:04:10 AM - Project '../common/tsconfig.json' is up to date because newest input '../common/src/utils/batches.ts' is older than output '../common/tsconfig.tsbuildinfo'
28
+ 9:05:45 AM - Project '../common/tsconfig.json' is up to date because newest input '../common/src/error.ts' is older than output '../common/tsconfig.tsbuildinfo'
29
29
 
30
- 11:04:10 AM - Project '../../dev/workspace/tsconfig.json' is up to date because newest input '../../dev/workspace/src/index.ts' is older than output '../../dev/workspace/tsconfig.tsbuildinfo'
30
+ 9:05:45 AM - Project '../../dev/workspace/tsconfig.json' is up to date because newest input '../../dev/workspace/src/projectInfo.ts' is older than output '../../dev/workspace/tsconfig.tsbuildinfo'
31
31
 
32
- 11:04:10 AM - Project 'tsconfig.json' is out of date because output file 'tsconfig.tsbuildinfo' does not exist
32
+ 9:05:45 AM - Project 'tsconfig.json' is out of date because output file 'tsconfig.tsbuildinfo' does not exist
33
33
 
34
- 11:04:10 AM - Building project '/home/runner/work/captcha/captcha/packages/datasets-fs/tsconfig.json'...
34
+ 9:05:45 AM - Building project '/home/runner/work/captcha/captcha/packages/datasets-fs/tsconfig.json'...
35
35
 
@@ -1,9 +1,9 @@
1
1
 
2
- > @prosopo/datasets-fs@3.0.136 build
2
+ > @prosopo/datasets-fs@3.0.138 build
3
3
  > npm run build:cross-env -- --mode ${NODE_ENV:-development}
4
4
 
5
5
 
6
- > @prosopo/datasets-fs@3.0.136 build:cross-env
6
+ > @prosopo/datasets-fs@3.0.138 build:cross-env
7
7
  > vite build --config vite.esm.config.ts --mode production
8
8
 
9
9
  ViteEsmConfig: .
@@ -11,23 +11,23 @@ ViteEsmConfig: .
11
11
  tsConfigPaths: [
12
12
  '/home/runner/work/captcha/captcha/packages/common/tsconfig.json',
13
13
  '/home/runner/work/captcha/captcha/packages/locale/tsconfig.json',
14
- '/home/runner/work/captcha/captcha/packages/logger/tsconfig.json',
15
14
  '/home/runner/work/captcha/captcha/packages/util/tsconfig.json',
15
+ '/home/runner/work/captcha/captcha/packages/logger/tsconfig.json',
16
16
  '/home/runner/work/captcha/captcha/packages/types/tsconfig.json',
17
17
  '/home/runner/work/captcha/captcha/packages/util-crypto/tsconfig.json'
18
18
  ]
19
19
  }
20
20
  {
21
21
  externals: [
22
+ The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the resolve.tsconfigPaths option. You can remove the plugin and set resolve.tsconfigPaths: true in your Vite config instead.
22
23
  '@prosopo/common',
23
24
  '@prosopo/locale',
24
- '@prosopo/logger',
25
25
  '@prosopo/util',
26
+ '@prosopo/logger',
26
27
  '@prosopo/types',
27
28
  '@prosopo/util-crypto'
28
29
  ]
29
30
  }
30
- The plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the resolve.tsconfigPaths option. You can remove the plugin and set resolve.tsconfigPaths: true in your Vite config instead.
31
31
  vite v8.1.5 building ssr environment for production...
32
32
  Bundle build started
33
33
  
@@ -38,7 +38,7 @@ dist/_virtual/_rolldown/runtime.js 0.04 kB │ gzip: 0.06 kB
38
38
  dist/index.js 0.09 kB │ gzip: 0.09 kB
39
39
  dist/cli/cli.js 1.52 kB │ gzip: 0.73 kB
40
40
 
41
- ✓ built in 58ms
41
+ ✓ built in 64ms
42
42
  [copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat/data.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat/data.json
43
43
  [copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat_resized/captchas_v1.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat_resized/captchas_v1.json
44
44
  [copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat_resized/captchas_v2.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat_resized/captchas_v2.json
package/CHANGELOG.md CHANGED
@@ -1,5 +1,37 @@
1
1
  # @prosopo/datasets-fs
2
2
 
3
+ ## 3.0.138
4
+ ### Patch Changes
5
+
6
+ - Updated dependencies [4c9b84b]
7
+ - @prosopo/types@5.10.1
8
+
9
+ ## 3.0.137
10
+ ### Patch Changes
11
+
12
+ - a9141c3: Ship one copy of each crypto library in the widget instead of two or three.
13
+
14
+ The bundle contained three separate copies of `@noble/hashes` and two of `@polkadot/util`, because different packages asked for different major versions and npm installed each one in its own folder. Same code, bundled repeatedly. The widget's eager payload drops by about 6KB gzipped.
15
+
16
+ Two stale version pins caused it:
17
+
18
+ - `@prosopo/util-crypto` asked for `@noble/hashes` 1.8.0 while its own dependencies `@noble/curves` and `@scure/sr25519` asked for 2.4.0, so npm installed both majors. Everything is now on 2.4.0. The v2 import paths changed (`@noble/hashes/sha256` is now `@noble/hashes/sha2.js`, `blake2b` is `blake2.js`); the functions themselves are unchanged.
19
+ - `@prosopo/util-crypto` asked for `@polkadot/x-randomvalues` 13.5.7, which in turn demands exactly `@polkadot/util` 13.5.7. Every other package in the repo asks for 14.0.3, so npm put the old one in the shared folder and gave each package its own private copy of the new one. Bumping that single pin to 14.0.3 leaves one shared copy. As a side effect `@prosopo/util-crypto` now gets the `@scure/base` 2.4.0 it always asked for, rather than the 1.2.6 it was silently given.
20
+
21
+ `@prosopo/keyring` no longer depends on `@polkadot/util-crypto`. It was used for one type, which `@prosopo/util-crypto` already exports.
22
+
23
+ The repo root now names the versions the workspace standardises on (`@noble/hashes`, `@noble/curves`, `@scure/base`, `@scure/sr25519`), which is what keeps npm putting them in the shared folder. Older majors are still installed for the Polkadot web3 packages that require them; those load only in web3 mode and are unaffected.
24
+
25
+ Covered by the existing test suites for each package, all passing unchanged.
26
+ - Updated dependencies [a9141c3]
27
+ - Updated dependencies [a9141c3]
28
+ - Updated dependencies [a9141c3]
29
+ - @prosopo/common@3.1.59
30
+ - @prosopo/types@5.10.0
31
+ - @prosopo/logger@2.1.0
32
+ - @prosopo/util-crypto@13.5.33
33
+ - @prosopo/util@3.3.11
34
+
3
35
  ## 3.0.136
4
36
  ### Patch Changes
5
37
 
package/README.md CHANGED
@@ -1,116 +1,154 @@
1
- # README
1
+ # @prosopo/datasets-fs
2
2
 
3
- ## How to produce captchas
3
+ Tooling for building the image dataset that an `image` CAPTCHA serves from.
4
4
 
5
- You need 3 files:
5
+ You need this only if you want image CAPTCHAs. `pow` needs no imagery, and `puzzle` generates its imagery
6
+ procedurally at request time. See the [self-hosting guide](https://docs.prosopo.io/en/self-hosting/) for the wider
7
+ context.
6
8
 
7
- 1. a json file containing labelled data in the form:
9
+ Build the CLI once, then run it from this package directory:
8
10
 
9
- ```json
10
- [
11
- {
12
- "label": "dog",
13
- "data": "http://example.com/a.png"
14
- },
15
- {
16
- "label": "cat",
17
- "data": "http://example.com/b.png"
18
- },
11
+ ```bash
12
+ npm run -w @prosopo/datasets-fs build
13
+ node dist/cli.js --help
14
+ ```
15
+
16
+ Commands: `flatten`, `labels`, `resize`, `relocate`, `get`, `generate-v1`, `generate-v2`. Every command takes
17
+ `--input`/`--in`, `--output`/`--out` and `--overwrite`; run `node dist/cli.js <command> --help` for the rest.
18
+
19
+ ## The images must be publicly reachable
20
+
21
+ A dataset does not embed images. Each entry holds a **URL**, and the browser fetches it while the user solves the
22
+ challenge. So you host the images yourself — any static host or CDN — and the dataset points at them. `relocate`
23
+ exists to rewrite local paths into those public URLs once you know them.
24
+
25
+ ## Building a dataset
26
+
27
+ Start from a directory with one subdirectory per label:
28
+
29
+ ```
30
+ data/
31
+ dog/
32
+ a.png
19
33
  ...
20
- ]
21
- ```
34
+ cat/
35
+ b.png
36
+ ...
37
+ ```
22
38
 
23
- 1. a json file containing unlabelled data in the form:
39
+ ### 1. Flatten
24
40
 
25
- ```json
26
- [
27
- {
28
- "data": "http://example.com/c.png"
29
- },
41
+ Collapses the tree into a single image directory plus a `data.json` mapping each image to its label. Images are
42
+ renamed to their content hash.
43
+
44
+ ```bash
45
+ node dist/cli.js flatten --in ./data --out ./flat --overwrite
46
+ ```
47
+
48
+ `./flat/data.json` looks like this — note the `items` wrapper, and that `data` is still a local path at this stage:
49
+
50
+ ```json
51
+ {
52
+ "items": [
30
53
  {
31
- "data": "http://example.com/d.png"
32
- },
33
- ...
54
+ "data": "/abs/path/flat/images/0x1640d689....png",
55
+ "type": "image",
56
+ "label": "bird",
57
+ "hash": "0x1640d689..."
58
+ }
34
59
  ]
35
- ```
60
+ }
61
+ ```
62
+
63
+ ### 2. Extract the labels
64
+
65
+ ```bash
66
+ node dist/cli.js labels --in ./flat/data.json --out ./labels.json --overwrite
67
+ ```
36
68
 
37
- 1. (optional) a json file with an array of labels which unlabelled data can be categorised into. If not specified will default to the same labels as seen in the labelled data.
69
+ ```json
70
+ { "labels": ["bird", "bus", "car", "cat", "deer", "dog", "horse", "plane", "train"] }
71
+ ```
38
72
 
39
- Then build and run the cli, passing appropriate parameters:
40
- `npm run build && node ./dist/js/cli.js generate --labelled /path/to/my/labelled/data.json --unlabelled /path/to/my/unlabelled/data.json --seed 0 --labels /path/to/my/labels.json --output /path/to/the/output/captchas.json`
73
+ ### 3. Resize (optional)
41
74
 
42
- Use `node ./dist/js/cli.ts --help` to inspect other parameters.
75
+ ```bash
76
+ node dist/cli.js resize --in ./flat/data.json --out ./resized --size 128 --square --overwrite
77
+ ```
43
78
 
44
- Commands:
79
+ ### 4. Point the entries at your host
45
80
 
46
- 1. `flatten` converts a hierarchical directory structure into a single directory with corresponding map file, e.g.
81
+ Upload the images, then rewrite the local paths to the URLs they now live at:
47
82
 
48
- ```json
49
- data/
50
- dog/
51
- a.png
52
- ...
53
- cat/
54
- b.png
55
- ...
56
- ```
83
+ ```bash
84
+ node dist/cli.js relocate \
85
+ --in ./flat/data.json \
86
+ --out ./data-hosted.json \
87
+ --from /abs/path/flat/images \
88
+ --to https://img.example.com \
89
+ --overwrite
90
+ ```
57
91
 
58
- into
92
+ Check every URL actually resolves before going further — `get` reports anything that 404s:
59
93
 
60
- ```json
61
- data/
62
- a.png
63
- b.png
64
- ...
65
- map.json
66
- ```
94
+ ```bash
95
+ node dist/cli.js get --in ./data-hosted.json
96
+ ```
67
97
 
68
- where `map.json` looks like:
98
+ ### 5. Generate the captchas
69
99
 
70
- ```json
71
- [
72
- {
73
- "label": "dog",
74
- "data": "http://example.com/a.png"
75
- },
76
- {
77
- "label": "cat",
78
- "data": "http://example.com/b.png"
79
- },
80
- ...
81
- ]
82
- ```
83
-
84
- e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js flatten --in ~/bench/test/data --out ~/bench/test/flat --overwrite`
85
- 1. `generateDistinct` takes the 3 files described above and produces captcha challenges comprising 2 rounds, one labelled and one unlabelled.
86
- e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js generate distinct --out ~/bench/test/captchas.json --labelled ~/bench/test/flat/data.json --unlabelled ~/bench/test/flat/data.json --seed 0 --size 9 --allow-duplicates --solved 1 --unsolved 1 --overwrite --labels ~/bench/test/flat/labels.json`
87
- 1. `generateUnion` takes the 3 files described above and produces captcha challenges comprising one or more rounds, mixing labelled and unlabelled data into a single round.
88
- e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js generate union --out ~/bench/test/captchas-union.json --labelled ~/bench/test/flat/data.json --unlabelled ~/bench/test/flat/data.json --seed 0 --size 9 --allow-duplicates --count 2 --overwrite --labels ~/bench/test/flat/labels.json`
89
- 1. 'labels' gets all labels from a data json.
90
- e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js labels --data ~/bench/test/flat/data2.json`
91
- 1. 'get' fetches all images using a GET request, displaying errors for any images which hit 404 or not OK.
92
- e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js get --data ~/bench/test/flat/data2.json`
93
- 1. 'relocate' rewrites the url of images in a json file.
94
- e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js relocate --from example.com --to web.site --data ~/bench/test/flat/data2.json`
95
- 1. 'scale' rescales images in a flat directory structure to a given size.
96
- e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js scale --data ~/bench/test/flat/data.json --out ~/bench/test/flat/images2 --overwrite --size 128`
97
-
98
- ## Generating test data
99
-
100
- The test data was generated using:
100
+ `generate-v2` mixes labelled and unlabelled images into a single round:
101
101
 
102
102
  ```bash
103
- cli generate-v1 --min-correct 1 --max-correct 6 --labelled $PWD/src/tests/data/flat_resized/data.json --unlabelled $PWD/src/tests/data/flat_resized/data.json --out $PWD/src/tests/data/flat_resized/captchas_v1.json --solved 50 --unsolved 50 --seed 0 --allowDuplicates --seed 0
103
+ node dist/cli.js generate-v2 \
104
+ --out ./captchas.json \
105
+ --labelled ./data-hosted.json \
106
+ --unlabelled ./data-hosted.json \
107
+ --labels ./labels.json \
108
+ --seed 0 \
109
+ --size 9 \
110
+ --count 100 \
111
+ --minCorrect 1 \
112
+ --minIncorrect 1 \
113
+ --minLabelled 2 \
114
+ --maxLabelled 7 \
115
+ --allowDuplicates \
116
+ --overwrite
117
+ ```
104
118
 
105
- cli generate-v2 --count 100 --min-correct 1 --min-incorrect 1 --min-labelled 2 --max-labelled 7 --labelled $PWD/src/tests/data/flat_resized/data.json --unlabelled $PWD/src/tests/data/flat_resized/data.json --out $PWD/src/tests/data/flat_resized/captchas_v2.json --seed 0 --allowDuplicates --seed 0
119
+ **`--count` is not optional in practice.** It defaults to zero, and without it the command exits successfully having
120
+ written a dataset containing no captchas at all. If your `captchas.json` has an empty `captchas` array, this is why.
106
121
 
107
- cli relocate --from '${repo}' --to newwebsite.com --input $PWD/src/tests/data/flat_resized/data.json --output $PWD/src/tests/data/flat_resized/relocated_data.json --overwrite
122
+ `generate-v1` is the older two-round format (one labelled round, one unlabelled). It takes `--solved` and `--unsolved`
123
+ counts instead of `--count`.
108
124
 
109
- cli labels --input $PWD/src/tests/data/flat_resized/data.json --output $PWD/src/tests/data/flat_resized/labels.json --overwrite
125
+ ## Loading it into a provider
126
+
127
+ ```bash
128
+ docker compose cp ./captchas.json provider:/usr/src/app/captchas.json
129
+ docker compose exec provider npx provider provider_set_data_set --file /usr/src/app/captchas.json
130
+ docker compose restart provider
131
+ ```
110
132
 
111
- cli flatten --in $PWD/src/tests/data/hierarchical --out $PWD/src/tests/data/flat --overwrite
133
+ **The restart is required.** The provider resolves its default dataset once, when the environment initialises, so a
134
+ dataset uploaded into a running provider is stored but not served — image challenges keep failing with
135
+ `No dataset available. Please upload a dataset first.` until the process restarts.
112
136
 
113
- cli resize --square --size 128 --in $PWD/src/tests/data/flat/data.json --out $PWD/src/tests/data/flat_resized --overwrite
137
+ Then register your site key for image captchas:
138
+
139
+ ```bash
140
+ docker compose exec provider npx provider site_key_register <sitekey> enterprise \
141
+ --captcha_type image --domains example.com \
142
+ --pow_difficulty 4 --frictionless_threshold 0.5 --image_threshold 0.8
114
143
  ```
115
144
 
116
- `cli` is an alias to `npm run build && npm run cli --`
145
+ ## Regenerating the test fixtures
146
+
147
+ The fixtures under `src/tests/data` were produced with:
148
+
149
+ ```bash
150
+ node dist/cli.js flatten --in ./src/tests/data/hierarchical --out ./src/tests/data/flat --overwrite
151
+ node dist/cli.js resize --square --size 128 --in ./src/tests/data/flat/data.json --out ./src/tests/data/flat_resized --overwrite
152
+ node dist/cli.js labels --in ./src/tests/data/flat_resized/data.json --out ./src/tests/data/flat_resized/labels.json --overwrite
153
+ node dist/cli.js relocate --from '${repo}' --to newwebsite.com --in ./src/tests/data/flat_resized/data.json --out ./src/tests/data/flat_resized/relocated_data.json --overwrite
154
+ ```
@@ -1,5 +1,5 @@
1
1
  import fs from "node:fs";
2
- import { blake2b } from "@noble/hashes/blake2b";
2
+ import { blake2b } from "@noble/hashes/blake2.js";
3
3
  import { ProsopoDatasetError, ProsopoEnvError } from "@prosopo/common";
4
4
  import { DataSchema } from "@prosopo/types";
5
5
  import { u8aToHex } from "@prosopo/util";
@@ -1 +1 @@
1
- {"version":3,"file":"resize.js","sourceRoot":"","sources":["../../src/commands/resize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAa,UAAU,EAAa,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAc1B,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACN,qBAAqB,EACrB,qBAAqB,GACrB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,CAAC,MAAM,UAAU,GAAG,qBAAqB,CAAC,MAAM,CAAC;IACtD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAIH,MAAM,OAAO,MAAO,SAAQ,qBAAqC;IAChD,YAAY;QAC3B,OAAO,UAAU,CAAC;IACnB,CAAC;IAEe,UAAU;QACzB,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE;YACzC,KAAK,EAAE;gBACN,WAAW,EACV,8DAA8D;aAC/D;YACD,MAAM,EAAE;gBACP,WAAW,EACV,wNAAwN;aACzN;YACD,IAAI,EAAE;gBACL,MAAM,EAAE,IAAI;gBACZ,WAAW,EACV,gJAAgJ;aACjJ;YACD,MAAM,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,WAAW,EACV,oJAAoJ;aACrJ;SACD,CAAC,CAAC;IACJ,CAAC;IAEe,KAAK,CAAC,IAAI,CAAC,IAAU;QACpC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAEpC,MAAM,OAAO,GAAW,IAAI,CAAC,KAAK,CAAC;QACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,4BAA4B,OAAO,EAAE,CAAC,EAChD;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;QAC1C,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,eAAe,CACxB,IAAI,KAAK,CAAC,oCAAoC,MAAM,EAAE,CAAC,EACvD;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QAGD,MAAM,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC;QAClC,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAG1C,MAAM,UAAU,GAAW,UAAU,CAAC,KAAK,CAC1C,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAC5C,CAAC,KAAK,CAAC;QAGR,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACpC,EAAE,EACF,WAAW,CAAC,OAAO,CAAC,cAAc,CAClC,CAAC;QACF,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,WAAW,GAAW,EAAE,CAAC;QAC/B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,GAAG,CAAC,SAAS,EAAE,CAAC;YAGhB,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC;iBAC9B,MAAM,CAAC;gBACP,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;aAChC,CAAC;iBACD,WAAW,EAAE;iBACb,YAAY,CAAC,MAAM,CAAC;iBACpB,IAAI,CAAC;gBACL,OAAO,EAAE,EAAE;gBACX,iBAAiB,EAAE,OAAO;gBAC1B,cAAc,EAAE,IAAI;gBACpB,mBAAmB,EAAE,IAAI;gBACzB,kBAAkB,EAAE,IAAI;gBACxB,aAAa,EAAE,IAAI;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,KAAK,EAAE,IAAI;aACX,CAAC,CAAC;YACJ,MAAM,WAAW,GAAG,GAAG,MAAM,UAAU,CAAC;YACxC,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAElC,MAAM,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;YAEhD,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;YACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAE3B,MAAM,aAAa,GAAG,GAAG,MAAM,IAAI,GAAG,MAAM,CAAC;YAC7C,EAAE,CAAC,UAAU,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;YAG1C,MAAM,UAAU,GAAS;gBACxB,GAAG,SAAS;gBACZ,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC;aACpC,CAAC;YACF,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;QACD,GAAG,CAAC,IAAI,EAAE,CAAC;QAGX,MAAM,aAAa,GAAG,GAAG,MAAM,YAAY,CAAC;QAE5C,MAAM,IAAI,GAAS;YAClB,KAAK,EAAE,WAAW;SAClB,CAAC;QAGF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC;QACpD,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;QAClD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YAC3D,SAAS,EAAE,IAAI;SACf,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;IAEe,cAAc;QAC7B,OAAO,+BAA+B,CAAC;IACxC,CAAC;CACD"}
1
+ {"version":3,"file":"resize.js","sourceRoot":"","sources":["../../src/commands/resize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAa,UAAU,EAAa,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAc1B,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACN,qBAAqB,EACrB,qBAAqB,GACrB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,CAAC,MAAM,UAAU,GAAG,qBAAqB,CAAC,MAAM,CAAC;IACtD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAIH,MAAM,OAAO,MAAO,SAAQ,qBAAqC;IAChD,YAAY;QAC3B,OAAO,UAAU,CAAC;IACnB,CAAC;IAEe,UAAU;QACzB,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE;YACzC,KAAK,EAAE;gBACN,WAAW,EACV,8DAA8D;aAC/D;YACD,MAAM,EAAE;gBACP,WAAW,EACV,wNAAwN;aACzN;YACD,IAAI,EAAE;gBACL,MAAM,EAAE,IAAI;gBACZ,WAAW,EACV,gJAAgJ;aACjJ;YACD,MAAM,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,WAAW,EACV,oJAAoJ;aACrJ;SACD,CAAC,CAAC;IACJ,CAAC;IAEe,KAAK,CAAC,IAAI,CAAC,IAAU;QACpC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAEpC,MAAM,OAAO,GAAW,IAAI,CAAC,KAAK,CAAC;QACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,4BAA4B,OAAO,EAAE,CAAC,EAChD;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;QAC1C,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,eAAe,CACxB,IAAI,KAAK,CAAC,oCAAoC,MAAM,EAAE,CAAC,EACvD;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QAGD,MAAM,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC;QAClC,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAG1C,MAAM,UAAU,GAAW,UAAU,CAAC,KAAK,CAC1C,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAC5C,CAAC,KAAK,CAAC;QAGR,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACpC,EAAE,EACF,WAAW,CAAC,OAAO,CAAC,cAAc,CAClC,CAAC;QACF,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,WAAW,GAAW,EAAE,CAAC;QAC/B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,GAAG,CAAC,SAAS,EAAE,CAAC;YAGhB,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC;iBAC9B,MAAM,CAAC;gBACP,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;aAChC,CAAC;iBACD,WAAW,EAAE;iBACb,YAAY,CAAC,MAAM,CAAC;iBACpB,IAAI,CAAC;gBACL,OAAO,EAAE,EAAE;gBACX,iBAAiB,EAAE,OAAO;gBAC1B,cAAc,EAAE,IAAI;gBACpB,mBAAmB,EAAE,IAAI;gBACzB,kBAAkB,EAAE,IAAI;gBACxB,aAAa,EAAE,IAAI;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,KAAK,EAAE,IAAI;aACX,CAAC,CAAC;YACJ,MAAM,WAAW,GAAG,GAAG,MAAM,UAAU,CAAC;YACxC,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAElC,MAAM,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;YAEhD,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;YACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAE3B,MAAM,aAAa,GAAG,GAAG,MAAM,IAAI,GAAG,MAAM,CAAC;YAC7C,EAAE,CAAC,UAAU,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;YAG1C,MAAM,UAAU,GAAS;gBACxB,GAAG,SAAS;gBACZ,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC;aACpC,CAAC;YACF,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;QACD,GAAG,CAAC,IAAI,EAAE,CAAC;QAGX,MAAM,aAAa,GAAG,GAAG,MAAM,YAAY,CAAC;QAE5C,MAAM,IAAI,GAAS;YAClB,KAAK,EAAE,WAAW;SAClB,CAAC;QAGF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC;QACpD,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;QAClD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YAC3D,SAAS,EAAE,IAAI;SACf,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;IAEe,cAAc;QAC7B,OAAO,+BAA+B,CAAC;IACxC,CAAC;CACD"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@prosopo/datasets-fs",
3
- "version": "3.0.136",
3
+ "version": "3.0.138",
4
4
  "author": "PROSOPO LIMITED <info@prosopo.io>",
5
5
  "license": "Apache-2.0",
6
6
  "private": false,
@@ -29,12 +29,12 @@
29
29
  }
30
30
  },
31
31
  "dependencies": {
32
- "@noble/hashes": "1.8.0",
33
- "@prosopo/common": "3.1.58",
34
- "@prosopo/logger": "2.0.10",
35
- "@prosopo/types": "5.9.2",
36
- "@prosopo/util": "3.3.10",
37
- "@prosopo/util-crypto": "13.5.32",
32
+ "@noble/hashes": "2.4.0",
33
+ "@prosopo/common": "3.1.59",
34
+ "@prosopo/logger": "2.1.0",
35
+ "@prosopo/types": "5.10.1",
36
+ "@prosopo/util": "3.3.11",
37
+ "@prosopo/util-crypto": "13.5.33",
38
38
  "@prosopo/workspace": "3.2.1",
39
39
  "bcrypt": "6.0.0",
40
40
  "cli-progress": "3.12.0",
@@ -43,7 +43,7 @@
43
43
  "zod": "3.23.8"
44
44
  },
45
45
  "devDependencies": {
46
- "@prosopo/config": "3.4.0",
46
+ "@prosopo/config": "3.4.1",
47
47
  "@types/bcrypt": "6.0.0",
48
48
  "@types/cli-progress": "3.11.6",
49
49
  "@types/node": "22.10.2",
@@ -1,5 +1,5 @@
1
1
  import fs from "node:fs";
2
- import { blake2b } from "@noble/hashes/blake2b";
2
+ import { blake2b } from "@noble/hashes/blake2.js";
3
3
  import { ProsopoDatasetError, ProsopoEnvError } from "@prosopo/common";
4
4
  import { type Data, DataSchema, type Item } from "@prosopo/types";
5
5
  import { u8aToHex } from "@prosopo/util";