@prosopo/datasets-fs 3.0.136 → 3.0.138
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build$colon$cjs.log +5 -5
- package/.turbo/turbo-build$colon$tsc.log +13 -13
- package/.turbo/turbo-build.log +6 -6
- package/CHANGELOG.md +32 -0
- package/README.md +126 -88
- package/dist/commands/resize.js +1 -1
- package/dist/commands/resize.js.map +1 -1
- package/package.json +8 -8
- package/src/commands/resize.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
|
|
2
|
-
> @prosopo/datasets-fs@3.0.
|
|
2
|
+
> @prosopo/datasets-fs@3.0.138 build:cjs
|
|
3
3
|
> NODE_ENV=${NODE_ENV:-development}; vite build --config vite.cjs.config.ts --mode $NODE_ENV
|
|
4
4
|
|
|
5
5
|
ViteCommonJSConfig: .
|
|
@@ -7,23 +7,23 @@ ViteCommonJSConfig: .
|
|
|
7
7
|
tsConfigPaths: [
|
|
8
8
|
'/home/runner/work/captcha/captcha/packages/common/tsconfig.json',
|
|
9
9
|
'/home/runner/work/captcha/captcha/packages/locale/tsconfig.json',
|
|
10
|
-
'/home/runner/work/captcha/captcha/packages/logger/tsconfig.json',
|
|
11
10
|
'/home/runner/work/captcha/captcha/packages/util/tsconfig.json',
|
|
11
|
+
'/home/runner/work/captcha/captcha/packages/logger/tsconfig.json',
|
|
12
12
|
'/home/runner/work/captcha/captcha/packages/types/tsconfig.json',
|
|
13
13
|
'/home/runner/work/captcha/captcha/packages/util-crypto/tsconfig.json'
|
|
14
14
|
]
|
|
15
15
|
}
|
|
16
16
|
{
|
|
17
17
|
externals: [
|
|
18
|
+
[33mThe plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the [1mresolve.tsconfigPaths[22m option. You can remove the plugin and set [1mresolve.tsconfigPaths: true[22m in your Vite config instead.[39m
|
|
18
19
|
'@prosopo/common',
|
|
19
20
|
'@prosopo/locale',
|
|
20
|
-
'@prosopo/logger',
|
|
21
21
|
'@prosopo/util',
|
|
22
|
+
'@prosopo/logger',
|
|
22
23
|
'@prosopo/types',
|
|
23
24
|
'@prosopo/util-crypto'
|
|
24
25
|
]
|
|
25
26
|
}
|
|
26
|
-
[33mThe plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the [1mresolve.tsconfigPaths[22m option. You can remove the plugin and set [1mresolve.tsconfigPaths: true[22m in your Vite config instead.[39m
|
|
27
27
|
[36mvite v8.1.5 [32mbuilding ssr environment for production...[36m[39m
|
|
28
28
|
Bundle build started
|
|
29
29
|
[2K
|
|
@@ -34,7 +34,7 @@ dist/cjs/index.cjs 0.15 kB │ gzip: 0.14 kB
|
|
|
34
34
|
dist/cjs/_virtual/_rolldown/runtime.cjs 1.00 kB │ gzip: 0.50 kB
|
|
35
35
|
dist/cjs/cli/cli.cjs 1.74 kB │ gzip: 0.81 kB
|
|
36
36
|
|
|
37
|
-
[32m✓ built in
|
|
37
|
+
[32m✓ built in 70ms[39m
|
|
38
38
|
[copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat/data.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat/data.json
|
|
39
39
|
[copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat_resized/captchas_v1.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat_resized/captchas_v1.json
|
|
40
40
|
[copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat_resized/captchas_v2.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat_resized/captchas_v2.json
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
|
|
2
|
-
> @prosopo/datasets-fs@3.0.
|
|
2
|
+
> @prosopo/datasets-fs@3.0.138 build:tsc
|
|
3
3
|
> tsc --build --verbose
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
9:05:45 AM - Projects in this build:
|
|
6
6
|
* ../../dev/config/tsconfig.json
|
|
7
|
-
* ../locale/tsconfig.json
|
|
8
7
|
* ../util/tsconfig.json
|
|
8
|
+
* ../locale/tsconfig.json
|
|
9
9
|
* ../logger/tsconfig.json
|
|
10
10
|
* ../util-crypto/tsconfig.json
|
|
11
11
|
* ../types/tsconfig.json
|
|
@@ -13,23 +13,23 @@
|
|
|
13
13
|
* ../../dev/workspace/tsconfig.json
|
|
14
14
|
* tsconfig.json
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
9:05:45 AM - Project '../../dev/config/tsconfig.json' is up to date because newest input '../../dev/config/src/webpack/webpack.config.ts' is older than output '../../dev/config/tsconfig.tsbuildinfo'
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
9:05:45 AM - Project '../util/tsconfig.json' is up to date because newest input '../util/src/url.ts' is older than output '../util/tsconfig.tsbuildinfo'
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
9:05:45 AM - Project '../locale/tsconfig.json' is up to date because newest input '../locale/src/translationKey.ts' is older than output '../locale/tsconfig.tsbuildinfo'
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
9:05:45 AM - Project '../logger/tsconfig.json' is up to date because newest input '../logger/src/index.ts' is older than output '../logger/tsconfig.tsbuildinfo'
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
9:05:45 AM - Project '../util-crypto/tsconfig.json' is up to date because newest input '../util-crypto/src/types.ts' is older than output '../util-crypto/tsconfig.tsbuildinfo'
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
9:05:45 AM - Project '../types/tsconfig.json' is up to date because newest input '../types/src/types.test-d.ts' is older than output '../types/tsconfig.tsbuildinfo'
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
9:05:45 AM - Project '../common/tsconfig.json' is up to date because newest input '../common/src/error.ts' is older than output '../common/tsconfig.tsbuildinfo'
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
9:05:45 AM - Project '../../dev/workspace/tsconfig.json' is up to date because newest input '../../dev/workspace/src/projectInfo.ts' is older than output '../../dev/workspace/tsconfig.tsbuildinfo'
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
9:05:45 AM - Project 'tsconfig.json' is out of date because output file 'tsconfig.tsbuildinfo' does not exist
|
|
33
33
|
|
|
34
|
-
|
|
34
|
+
9:05:45 AM - Building project '/home/runner/work/captcha/captcha/packages/datasets-fs/tsconfig.json'...
|
|
35
35
|
|
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
|
|
2
|
-
> @prosopo/datasets-fs@3.0.
|
|
2
|
+
> @prosopo/datasets-fs@3.0.138 build
|
|
3
3
|
> npm run build:cross-env -- --mode ${NODE_ENV:-development}
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
> @prosopo/datasets-fs@3.0.
|
|
6
|
+
> @prosopo/datasets-fs@3.0.138 build:cross-env
|
|
7
7
|
> vite build --config vite.esm.config.ts --mode production
|
|
8
8
|
|
|
9
9
|
ViteEsmConfig: .
|
|
@@ -11,23 +11,23 @@ ViteEsmConfig: .
|
|
|
11
11
|
tsConfigPaths: [
|
|
12
12
|
'/home/runner/work/captcha/captcha/packages/common/tsconfig.json',
|
|
13
13
|
'/home/runner/work/captcha/captcha/packages/locale/tsconfig.json',
|
|
14
|
-
'/home/runner/work/captcha/captcha/packages/logger/tsconfig.json',
|
|
15
14
|
'/home/runner/work/captcha/captcha/packages/util/tsconfig.json',
|
|
15
|
+
'/home/runner/work/captcha/captcha/packages/logger/tsconfig.json',
|
|
16
16
|
'/home/runner/work/captcha/captcha/packages/types/tsconfig.json',
|
|
17
17
|
'/home/runner/work/captcha/captcha/packages/util-crypto/tsconfig.json'
|
|
18
18
|
]
|
|
19
19
|
}
|
|
20
20
|
{
|
|
21
21
|
externals: [
|
|
22
|
+
[33mThe plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the [1mresolve.tsconfigPaths[22m option. You can remove the plugin and set [1mresolve.tsconfigPaths: true[22m in your Vite config instead.[39m
|
|
22
23
|
'@prosopo/common',
|
|
23
24
|
'@prosopo/locale',
|
|
24
|
-
'@prosopo/logger',
|
|
25
25
|
'@prosopo/util',
|
|
26
|
+
'@prosopo/logger',
|
|
26
27
|
'@prosopo/types',
|
|
27
28
|
'@prosopo/util-crypto'
|
|
28
29
|
]
|
|
29
30
|
}
|
|
30
|
-
[33mThe plugin "vite-tsconfig-paths" is detected. Vite now supports tsconfig paths resolution natively via the [1mresolve.tsconfigPaths[22m option. You can remove the plugin and set [1mresolve.tsconfigPaths: true[22m in your Vite config instead.[39m
|
|
31
31
|
[36mvite v8.1.5 [32mbuilding ssr environment for production...[36m[39m
|
|
32
32
|
Bundle build started
|
|
33
33
|
[2K
|
|
@@ -38,7 +38,7 @@ dist/_virtual/_rolldown/runtime.js 0.04 kB │ gzip: 0.06 kB
|
|
|
38
38
|
dist/index.js 0.09 kB │ gzip: 0.09 kB
|
|
39
39
|
dist/cli/cli.js 1.52 kB │ gzip: 0.73 kB
|
|
40
40
|
|
|
41
|
-
[32m✓ built in
|
|
41
|
+
[32m✓ built in 64ms[39m
|
|
42
42
|
[copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat/data.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat/data.json
|
|
43
43
|
[copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat_resized/captchas_v1.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat_resized/captchas_v1.json
|
|
44
44
|
[copy-plugin] copying /home/runner/work/captcha/captcha/packages/datasets-fs/src/tests/data/flat_resized/captchas_v2.json to /home/runner/work/captcha/captcha/packages/datasets-fs/dist/tests/data/flat_resized/captchas_v2.json
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,37 @@
|
|
|
1
1
|
# @prosopo/datasets-fs
|
|
2
2
|
|
|
3
|
+
## 3.0.138
|
|
4
|
+
### Patch Changes
|
|
5
|
+
|
|
6
|
+
- Updated dependencies [4c9b84b]
|
|
7
|
+
- @prosopo/types@5.10.1
|
|
8
|
+
|
|
9
|
+
## 3.0.137
|
|
10
|
+
### Patch Changes
|
|
11
|
+
|
|
12
|
+
- a9141c3: Ship one copy of each crypto library in the widget instead of two or three.
|
|
13
|
+
|
|
14
|
+
The bundle contained three separate copies of `@noble/hashes` and two of `@polkadot/util`, because different packages asked for different major versions and npm installed each one in its own folder. Same code, bundled repeatedly. The widget's eager payload drops by about 6KB gzipped.
|
|
15
|
+
|
|
16
|
+
Two stale version pins caused it:
|
|
17
|
+
|
|
18
|
+
- `@prosopo/util-crypto` asked for `@noble/hashes` 1.8.0 while its own dependencies `@noble/curves` and `@scure/sr25519` asked for 2.4.0, so npm installed both majors. Everything is now on 2.4.0. The v2 import paths changed (`@noble/hashes/sha256` is now `@noble/hashes/sha2.js`, `blake2b` is `blake2.js`); the functions themselves are unchanged.
|
|
19
|
+
- `@prosopo/util-crypto` asked for `@polkadot/x-randomvalues` 13.5.7, which in turn demands exactly `@polkadot/util` 13.5.7. Every other package in the repo asks for 14.0.3, so npm put the old one in the shared folder and gave each package its own private copy of the new one. Bumping that single pin to 14.0.3 leaves one shared copy. As a side effect `@prosopo/util-crypto` now gets the `@scure/base` 2.4.0 it always asked for, rather than the 1.2.6 it was silently given.
|
|
20
|
+
|
|
21
|
+
`@prosopo/keyring` no longer depends on `@polkadot/util-crypto`. It was used for one type, which `@prosopo/util-crypto` already exports.
|
|
22
|
+
|
|
23
|
+
The repo root now names the versions the workspace standardises on (`@noble/hashes`, `@noble/curves`, `@scure/base`, `@scure/sr25519`), which is what keeps npm putting them in the shared folder. Older majors are still installed for the Polkadot web3 packages that require them; those load only in web3 mode and are unaffected.
|
|
24
|
+
|
|
25
|
+
Covered by the existing test suites for each package, all passing unchanged.
|
|
26
|
+
- Updated dependencies [a9141c3]
|
|
27
|
+
- Updated dependencies [a9141c3]
|
|
28
|
+
- Updated dependencies [a9141c3]
|
|
29
|
+
- @prosopo/common@3.1.59
|
|
30
|
+
- @prosopo/types@5.10.0
|
|
31
|
+
- @prosopo/logger@2.1.0
|
|
32
|
+
- @prosopo/util-crypto@13.5.33
|
|
33
|
+
- @prosopo/util@3.3.11
|
|
34
|
+
|
|
3
35
|
## 3.0.136
|
|
4
36
|
### Patch Changes
|
|
5
37
|
|
package/README.md
CHANGED
|
@@ -1,116 +1,154 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @prosopo/datasets-fs
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Tooling for building the image dataset that an `image` CAPTCHA serves from.
|
|
4
4
|
|
|
5
|
-
You need
|
|
5
|
+
You need this only if you want image CAPTCHAs. `pow` needs no imagery, and `puzzle` generates its imagery
|
|
6
|
+
procedurally at request time. See the [self-hosting guide](https://docs.prosopo.io/en/self-hosting/) for the wider
|
|
7
|
+
context.
|
|
6
8
|
|
|
7
|
-
|
|
9
|
+
Build the CLI once, then run it from this package directory:
|
|
8
10
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
11
|
+
```bash
|
|
12
|
+
npm run -w @prosopo/datasets-fs build
|
|
13
|
+
node dist/cli.js --help
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Commands: `flatten`, `labels`, `resize`, `relocate`, `get`, `generate-v1`, `generate-v2`. Every command takes
|
|
17
|
+
`--input`/`--in`, `--output`/`--out` and `--overwrite`; run `node dist/cli.js <command> --help` for the rest.
|
|
18
|
+
|
|
19
|
+
## The images must be publicly reachable
|
|
20
|
+
|
|
21
|
+
A dataset does not embed images. Each entry holds a **URL**, and the browser fetches it while the user solves the
|
|
22
|
+
challenge. So you host the images yourself — any static host or CDN — and the dataset points at them. `relocate`
|
|
23
|
+
exists to rewrite local paths into those public URLs once you know them.
|
|
24
|
+
|
|
25
|
+
## Building a dataset
|
|
26
|
+
|
|
27
|
+
Start from a directory with one subdirectory per label:
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
data/
|
|
31
|
+
dog/
|
|
32
|
+
a.png
|
|
19
33
|
...
|
|
20
|
-
|
|
21
|
-
|
|
34
|
+
cat/
|
|
35
|
+
b.png
|
|
36
|
+
...
|
|
37
|
+
```
|
|
22
38
|
|
|
23
|
-
1.
|
|
39
|
+
### 1. Flatten
|
|
24
40
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
41
|
+
Collapses the tree into a single image directory plus a `data.json` mapping each image to its label. Images are
|
|
42
|
+
renamed to their content hash.
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
node dist/cli.js flatten --in ./data --out ./flat --overwrite
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
`./flat/data.json` looks like this — note the `items` wrapper, and that `data` is still a local path at this stage:
|
|
49
|
+
|
|
50
|
+
```json
|
|
51
|
+
{
|
|
52
|
+
"items": [
|
|
30
53
|
{
|
|
31
|
-
"data": "
|
|
32
|
-
|
|
33
|
-
|
|
54
|
+
"data": "/abs/path/flat/images/0x1640d689....png",
|
|
55
|
+
"type": "image",
|
|
56
|
+
"label": "bird",
|
|
57
|
+
"hash": "0x1640d689..."
|
|
58
|
+
}
|
|
34
59
|
]
|
|
35
|
-
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### 2. Extract the labels
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
node dist/cli.js labels --in ./flat/data.json --out ./labels.json --overwrite
|
|
67
|
+
```
|
|
36
68
|
|
|
37
|
-
|
|
69
|
+
```json
|
|
70
|
+
{ "labels": ["bird", "bus", "car", "cat", "deer", "dog", "horse", "plane", "train"] }
|
|
71
|
+
```
|
|
38
72
|
|
|
39
|
-
|
|
40
|
-
`npm run build && node ./dist/js/cli.js generate --labelled /path/to/my/labelled/data.json --unlabelled /path/to/my/unlabelled/data.json --seed 0 --labels /path/to/my/labels.json --output /path/to/the/output/captchas.json`
|
|
73
|
+
### 3. Resize (optional)
|
|
41
74
|
|
|
42
|
-
|
|
75
|
+
```bash
|
|
76
|
+
node dist/cli.js resize --in ./flat/data.json --out ./resized --size 128 --square --overwrite
|
|
77
|
+
```
|
|
43
78
|
|
|
44
|
-
|
|
79
|
+
### 4. Point the entries at your host
|
|
45
80
|
|
|
46
|
-
|
|
81
|
+
Upload the images, then rewrite the local paths to the URLs they now live at:
|
|
47
82
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
```
|
|
83
|
+
```bash
|
|
84
|
+
node dist/cli.js relocate \
|
|
85
|
+
--in ./flat/data.json \
|
|
86
|
+
--out ./data-hosted.json \
|
|
87
|
+
--from /abs/path/flat/images \
|
|
88
|
+
--to https://img.example.com \
|
|
89
|
+
--overwrite
|
|
90
|
+
```
|
|
57
91
|
|
|
58
|
-
|
|
92
|
+
Check every URL actually resolves before going further — `get` reports anything that 404s:
|
|
59
93
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
b.png
|
|
64
|
-
...
|
|
65
|
-
map.json
|
|
66
|
-
```
|
|
94
|
+
```bash
|
|
95
|
+
node dist/cli.js get --in ./data-hosted.json
|
|
96
|
+
```
|
|
67
97
|
|
|
68
|
-
|
|
98
|
+
### 5. Generate the captchas
|
|
69
99
|
|
|
70
|
-
|
|
71
|
-
[
|
|
72
|
-
{
|
|
73
|
-
"label": "dog",
|
|
74
|
-
"data": "http://example.com/a.png"
|
|
75
|
-
},
|
|
76
|
-
{
|
|
77
|
-
"label": "cat",
|
|
78
|
-
"data": "http://example.com/b.png"
|
|
79
|
-
},
|
|
80
|
-
...
|
|
81
|
-
]
|
|
82
|
-
```
|
|
83
|
-
|
|
84
|
-
e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js flatten --in ~/bench/test/data --out ~/bench/test/flat --overwrite`
|
|
85
|
-
1. `generateDistinct` takes the 3 files described above and produces captcha challenges comprising 2 rounds, one labelled and one unlabelled.
|
|
86
|
-
e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js generate distinct --out ~/bench/test/captchas.json --labelled ~/bench/test/flat/data.json --unlabelled ~/bench/test/flat/data.json --seed 0 --size 9 --allow-duplicates --solved 1 --unsolved 1 --overwrite --labels ~/bench/test/flat/labels.json`
|
|
87
|
-
1. `generateUnion` takes the 3 files described above and produces captcha challenges comprising one or more rounds, mixing labelled and unlabelled data into a single round.
|
|
88
|
-
e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js generate union --out ~/bench/test/captchas-union.json --labelled ~/bench/test/flat/data.json --unlabelled ~/bench/test/flat/data.json --seed 0 --size 9 --allow-duplicates --count 2 --overwrite --labels ~/bench/test/flat/labels.json`
|
|
89
|
-
1. 'labels' gets all labels from a data json.
|
|
90
|
-
e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js labels --data ~/bench/test/flat/data2.json`
|
|
91
|
-
1. 'get' fetches all images using a GET request, displaying errors for any images which hit 404 or not OK.
|
|
92
|
-
e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js get --data ~/bench/test/flat/data2.json`
|
|
93
|
-
1. 'relocate' rewrites the url of images in a json file.
|
|
94
|
-
e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js relocate --from example.com --to web.site --data ~/bench/test/flat/data2.json`
|
|
95
|
-
1. 'scale' rescales images in a flat directory structure to a given size.
|
|
96
|
-
e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js scale --data ~/bench/test/flat/data.json --out ~/bench/test/flat/images2 --overwrite --size 128`
|
|
97
|
-
|
|
98
|
-
## Generating test data
|
|
99
|
-
|
|
100
|
-
The test data was generated using:
|
|
100
|
+
`generate-v2` mixes labelled and unlabelled images into a single round:
|
|
101
101
|
|
|
102
102
|
```bash
|
|
103
|
-
cli generate-
|
|
103
|
+
node dist/cli.js generate-v2 \
|
|
104
|
+
--out ./captchas.json \
|
|
105
|
+
--labelled ./data-hosted.json \
|
|
106
|
+
--unlabelled ./data-hosted.json \
|
|
107
|
+
--labels ./labels.json \
|
|
108
|
+
--seed 0 \
|
|
109
|
+
--size 9 \
|
|
110
|
+
--count 100 \
|
|
111
|
+
--minCorrect 1 \
|
|
112
|
+
--minIncorrect 1 \
|
|
113
|
+
--minLabelled 2 \
|
|
114
|
+
--maxLabelled 7 \
|
|
115
|
+
--allowDuplicates \
|
|
116
|
+
--overwrite
|
|
117
|
+
```
|
|
104
118
|
|
|
105
|
-
|
|
119
|
+
**`--count` is not optional in practice.** It defaults to zero, and without it the command exits successfully having
|
|
120
|
+
written a dataset containing no captchas at all. If your `captchas.json` has an empty `captchas` array, this is why.
|
|
106
121
|
|
|
107
|
-
|
|
122
|
+
`generate-v1` is the older two-round format (one labelled round, one unlabelled). It takes `--solved` and `--unsolved`
|
|
123
|
+
counts instead of `--count`.
|
|
108
124
|
|
|
109
|
-
|
|
125
|
+
## Loading it into a provider
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
docker compose cp ./captchas.json provider:/usr/src/app/captchas.json
|
|
129
|
+
docker compose exec provider npx provider provider_set_data_set --file /usr/src/app/captchas.json
|
|
130
|
+
docker compose restart provider
|
|
131
|
+
```
|
|
110
132
|
|
|
111
|
-
|
|
133
|
+
**The restart is required.** The provider resolves its default dataset once, when the environment initialises, so a
|
|
134
|
+
dataset uploaded into a running provider is stored but not served — image challenges keep failing with
|
|
135
|
+
`No dataset available. Please upload a dataset first.` until the process restarts.
|
|
112
136
|
|
|
113
|
-
|
|
137
|
+
Then register your site key for image captchas:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
docker compose exec provider npx provider site_key_register <sitekey> enterprise \
|
|
141
|
+
--captcha_type image --domains example.com \
|
|
142
|
+
--pow_difficulty 4 --frictionless_threshold 0.5 --image_threshold 0.8
|
|
114
143
|
```
|
|
115
144
|
|
|
116
|
-
|
|
145
|
+
## Regenerating the test fixtures
|
|
146
|
+
|
|
147
|
+
The fixtures under `src/tests/data` were produced with:
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
node dist/cli.js flatten --in ./src/tests/data/hierarchical --out ./src/tests/data/flat --overwrite
|
|
151
|
+
node dist/cli.js resize --square --size 128 --in ./src/tests/data/flat/data.json --out ./src/tests/data/flat_resized --overwrite
|
|
152
|
+
node dist/cli.js labels --in ./src/tests/data/flat_resized/data.json --out ./src/tests/data/flat_resized/labels.json --overwrite
|
|
153
|
+
node dist/cli.js relocate --from '${repo}' --to newwebsite.com --in ./src/tests/data/flat_resized/data.json --out ./src/tests/data/flat_resized/relocated_data.json --overwrite
|
|
154
|
+
```
|
package/dist/commands/resize.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
|
-
import { blake2b } from "@noble/hashes/
|
|
2
|
+
import { blake2b } from "@noble/hashes/blake2.js";
|
|
3
3
|
import { ProsopoDatasetError, ProsopoEnvError } from "@prosopo/common";
|
|
4
4
|
import { DataSchema } from "@prosopo/types";
|
|
5
5
|
import { u8aToHex } from "@prosopo/util";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resize.js","sourceRoot":"","sources":["../../src/commands/resize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,OAAO,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"resize.js","sourceRoot":"","sources":["../../src/commands/resize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAa,UAAU,EAAa,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAc1B,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACN,qBAAqB,EACrB,qBAAqB,GACrB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,CAAC,MAAM,UAAU,GAAG,qBAAqB,CAAC,MAAM,CAAC;IACtD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAIH,MAAM,OAAO,MAAO,SAAQ,qBAAqC;IAChD,YAAY;QAC3B,OAAO,UAAU,CAAC;IACnB,CAAC;IAEe,UAAU;QACzB,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE;YACzC,KAAK,EAAE;gBACN,WAAW,EACV,8DAA8D;aAC/D;YACD,MAAM,EAAE;gBACP,WAAW,EACV,wNAAwN;aACzN;YACD,IAAI,EAAE;gBACL,MAAM,EAAE,IAAI;gBACZ,WAAW,EACV,gJAAgJ;aACjJ;YACD,MAAM,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,WAAW,EACV,oJAAoJ;aACrJ;SACD,CAAC,CAAC;IACJ,CAAC;IAEe,KAAK,CAAC,IAAI,CAAC,IAAU;QACpC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAEpC,MAAM,OAAO,GAAW,IAAI,CAAC,KAAK,CAAC;QACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,4BAA4B,OAAO,EAAE,CAAC,EAChD;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;QAC1C,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,eAAe,CACxB,IAAI,KAAK,CAAC,oCAAoC,MAAM,EAAE,CAAC,EACvD;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QAGD,MAAM,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC;QAClC,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAG1C,MAAM,UAAU,GAAW,UAAU,CAAC,KAAK,CAC1C,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAC5C,CAAC,KAAK,CAAC;QAGR,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACpC,EAAE,EACF,WAAW,CAAC,OAAO,CAAC,cAAc,CAClC,CAAC;QACF,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,WAAW,GAAW,EAAE,CAAC;QAC/B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,GAAG,CAAC,SAAS,EAAE,CAAC;YAGhB,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC;iBAC9B,MAAM,CAAC;gBACP,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;aAChC,CAAC;iBACD,WAAW,EAAE;iBACb,YAAY,CAAC,MAAM,CAAC;iBACpB,IAAI,CAAC;gBACL,OAAO,EAAE,EAAE;gBACX,iBAAiB,EAAE,OAAO;gBAC1B,cAAc,EAAE,IAAI;gBACpB,mBAAmB,EAAE,IAAI;gBACzB,kBAAkB,EAAE,IAAI;gBACxB,aAAa,EAAE,IAAI;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,KAAK,EAAE,IAAI;aACX,CAAC,CAAC;YACJ,MAAM,WAAW,GAAG,GAAG,MAAM,UAAU,CAAC;YACxC,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAElC,MAAM,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;YAEhD,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;YACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAE3B,MAAM,aAAa,GAAG,GAAG,MAAM,IAAI,GAAG,MAAM,CAAC;YAC7C,EAAE,CAAC,UAAU,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;YAG1C,MAAM,UAAU,GAAS;gBACxB,GAAG,SAAS;gBACZ,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC;aACpC,CAAC;YACF,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;QACD,GAAG,CAAC,IAAI,EAAE,CAAC;QAGX,MAAM,aAAa,GAAG,GAAG,MAAM,YAAY,CAAC;QAE5C,MAAM,IAAI,GAAS;YAClB,KAAK,EAAE,WAAW;SAClB,CAAC;QAGF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC;QACpD,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;QAClD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YAC3D,SAAS,EAAE,IAAI;SACf,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;IAEe,cAAc;QAC7B,OAAO,+BAA+B,CAAC;IACxC,CAAC;CACD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@prosopo/datasets-fs",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.138",
|
|
4
4
|
"author": "PROSOPO LIMITED <info@prosopo.io>",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"private": false,
|
|
@@ -29,12 +29,12 @@
|
|
|
29
29
|
}
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@noble/hashes": "
|
|
33
|
-
"@prosopo/common": "3.1.
|
|
34
|
-
"@prosopo/logger": "2.0
|
|
35
|
-
"@prosopo/types": "5.
|
|
36
|
-
"@prosopo/util": "3.3.
|
|
37
|
-
"@prosopo/util-crypto": "13.5.
|
|
32
|
+
"@noble/hashes": "2.4.0",
|
|
33
|
+
"@prosopo/common": "3.1.59",
|
|
34
|
+
"@prosopo/logger": "2.1.0",
|
|
35
|
+
"@prosopo/types": "5.10.1",
|
|
36
|
+
"@prosopo/util": "3.3.11",
|
|
37
|
+
"@prosopo/util-crypto": "13.5.33",
|
|
38
38
|
"@prosopo/workspace": "3.2.1",
|
|
39
39
|
"bcrypt": "6.0.0",
|
|
40
40
|
"cli-progress": "3.12.0",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
"zod": "3.23.8"
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
|
-
"@prosopo/config": "3.4.
|
|
46
|
+
"@prosopo/config": "3.4.1",
|
|
47
47
|
"@types/bcrypt": "6.0.0",
|
|
48
48
|
"@types/cli-progress": "3.11.6",
|
|
49
49
|
"@types/node": "22.10.2",
|
package/src/commands/resize.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
|
-
import { blake2b } from "@noble/hashes/
|
|
2
|
+
import { blake2b } from "@noble/hashes/blake2.js";
|
|
3
3
|
import { ProsopoDatasetError, ProsopoEnvError } from "@prosopo/common";
|
|
4
4
|
import { type Data, DataSchema, type Item } from "@prosopo/types";
|
|
5
5
|
import { u8aToHex } from "@prosopo/util";
|