@apify/actor-utils 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -0
- package/package.json +63 -0
package/README.md
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# safePushData
|
|
2
|
+
|
|
3
|
+
TypeScript library wrapper that survives Apify dataset schema-validation
|
|
4
|
+
failures. When an upstream data source produces an item that doesn't match
|
|
5
|
+
the dataset's JSON schema, the platform rejects the **entire batch** with
|
|
6
|
+
a 400 error — losing every other valid item in that push. `safePushData`
|
|
7
|
+
parses the validation error, strips the offending fields, and retries.
|
|
8
|
+
|
|
9
|
+
## Usage
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
import { Actor } from 'apify';
|
|
13
|
+
import { safePushData } from 'apify-actor-utils';
|
|
14
|
+
|
|
15
|
+
await Actor.init();
|
|
16
|
+
|
|
17
|
+
const result = await safePushData((batch) => Actor.pushData(batch), items);
|
|
18
|
+
|
|
19
|
+
console.log(result);
|
|
20
|
+
// { pushed: 2, dropped: [...], attempts: 2 }
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Accepts a single item or an array. `pushFn` is the first positional arg
|
|
24
|
+
and is required — the library itself never imports the Apify SDK, and a
|
|
25
|
+
CI check forbids `.pushData(` from appearing anywhere in the source.
|
|
26
|
+
|
|
27
|
+
## Performance notes
|
|
28
|
+
|
|
29
|
+
The happy path is a single `try/await pushFn(items)` with **no extra
|
|
30
|
+
allocations** — no wrapper objects, no working copies, no maps. Only when
|
|
31
|
+
a `schema-validation-error` is caught does the wrapper materialise the
|
|
32
|
+
working state needed to clean and retry.
|
|
33
|
+
|
|
34
|
+
## How cleaning works
|
|
35
|
+
|
|
36
|
+
When the push fails with a `schema-validation-error`, the wrapper inspects
|
|
37
|
+
every AJV error per item:
|
|
38
|
+
|
|
39
|
+
| Error | Action |
|
|
40
|
+
| ------------------------------------ | ---------------------------------------------------------------------------------- |
|
|
41
|
+
| `required` at the root | Set the missing field to `null` and mark the path as a placeholder. |
|
|
42
|
+
| `additionalProperties` at the root | Delete the unknown property. |
|
|
43
|
+
| `type` / `format` / etc. at the root | Item itself is the wrong shape → dropped. |
|
|
44
|
+
| Constraint on a **placeholder** path | Replace with a type-aware default (see below). If no default is known → dropped. |
|
|
45
|
+
| Constraint on **user-supplied** data | Delete the field. If the schema later marks it required, a placeholder takes over. |
|
|
46
|
+
|
|
47
|
+
### Placeholder defaults
|
|
48
|
+
|
|
49
|
+
When a constraint fires on a path we placeholder'd ourselves, the wrapper
|
|
50
|
+
picks a value that should satisfy it:
|
|
51
|
+
|
|
52
|
+
| AJV keyword | Placeholder value |
|
|
53
|
+
| ------------------------------------------------------- | ------------------------------- |
|
|
54
|
+
| `type: string` | `''` |
|
|
55
|
+
| `type: integer` / `number` | `0` |
|
|
56
|
+
| `type: boolean` | `false` |
|
|
57
|
+
| `type: array` | `[]` |
|
|
58
|
+
| `type: object` | `{}` |
|
|
59
|
+
| `type: null` | `null` |
|
|
60
|
+
| `minLength: N` / `maxLength` | `'_'.repeat(N)` / `''` |
|
|
61
|
+
| `minimum: N` / `maximum` | `N` |
|
|
62
|
+
| `exclusiveMinimum: N` / `exclusiveMaximum` | `N + 1` / `N - 1` |
|
|
63
|
+
| `enum` | First allowed value |
|
|
64
|
+
| `format: email` / `uri` / `date` / `date-time` / `uuid` | a static valid example for each |
|
|
65
|
+
| Anything else (`pattern`, custom formats…) | Item is dropped. |
|
|
66
|
+
|
|
67
|
+
The retry loop chases one layer of errors per round
|
|
68
|
+
(`required` → `type` → `minLength` → push) until either the push succeeds
|
|
69
|
+
or `maxAttempts` (default 5) is hit.
|
|
70
|
+
|
|
71
|
+
## Options
|
|
72
|
+
|
|
73
|
+
| Option | Type | Default | Notes |
|
|
74
|
+
| ------------- | -------- | ------- | -------------------- |
|
|
75
|
+
| `maxAttempts` | `number` | `5` | Hard cap on retries. |
|
|
76
|
+
|
|
77
|
+
## Return shape
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
interface SafePushDataResult<T> {
|
|
81
|
+
pushed: number;
|
|
82
|
+
dropped: { item: T; errors: ValidationError[] }[];
|
|
83
|
+
attempts: number;
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## CI
|
|
88
|
+
|
|
89
|
+
Run `npm run ci` (typecheck + lint + format check + push-data guard + tests).
|
|
90
|
+
Highlights:
|
|
91
|
+
|
|
92
|
+
- `scripts/check-pushdata.mjs` greps `src/` and `test/` for any
|
|
93
|
+
`.pushData(` call expression and fails if one exists.
|
|
94
|
+
- The package is consumed as compiled output (`dist/`), so `npm test`
|
|
95
|
+
builds via `tsc` first, then runs the compiled suite under `node --test`.
|
|
96
|
+
|
|
97
|
+
## Layout
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
.
|
|
101
|
+
├── index.ts # package entry point, re-exports src/
|
|
102
|
+
├── src/safePushData.ts # the library
|
|
103
|
+
├── test/safePushData.test.ts # node:test suite
|
|
104
|
+
├── scripts/
|
|
105
|
+
│ ├── check-pushdata.mjs # CI guard against direct .pushData() calls
|
|
106
|
+
│ └── probe-errors.mjs # reference: re-derive the API error shape
|
|
107
|
+
├── tsconfig.json
|
|
108
|
+
└── package.json
|
|
109
|
+
```
|
package/package.json
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@apify/actor-utils",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Wrapper around Actor.pushData that survives dataset schema-validation failures by stripping offending fields and retrying. Callers pass the actual push function as pushFn.",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "https://github.com/apify-projects/apify-actor-utils.git"
|
|
9
|
+
},
|
|
10
|
+
"engines": {
|
|
11
|
+
"node": ">=22.6.0"
|
|
12
|
+
},
|
|
13
|
+
"types": "./dist/index.d.ts",
|
|
14
|
+
"main": "./dist/index.js",
|
|
15
|
+
"exports": {
|
|
16
|
+
".": {
|
|
17
|
+
"types": "./dist/index.d.ts",
|
|
18
|
+
"default": "./dist/index.js"
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"files": [
|
|
22
|
+
"dist/index.js",
|
|
23
|
+
"dist/index.js.map",
|
|
24
|
+
"dist/index.d.ts",
|
|
25
|
+
"dist/index.d.ts.map",
|
|
26
|
+
"dist/src"
|
|
27
|
+
],
|
|
28
|
+
"scripts": {
|
|
29
|
+
"build": "tsc",
|
|
30
|
+
"typecheck": "tsc --noEmit",
|
|
31
|
+
"lint": "eslint",
|
|
32
|
+
"lint:fix": "eslint --fix",
|
|
33
|
+
"format": "prettier --write .",
|
|
34
|
+
"format:check": "prettier --check .",
|
|
35
|
+
"test": "tsc && node --test dist/test/*.test.js",
|
|
36
|
+
"check:pushdata": "node scripts/check-pushdata.mjs",
|
|
37
|
+
"check-unused": "knip",
|
|
38
|
+
"ci": "npm run typecheck && npm run lint && npm run format:check && npm run check:pushdata && npm test",
|
|
39
|
+
"probe": "node scripts/probe-errors.mjs",
|
|
40
|
+
"prepare": "husky .husky || true"
|
|
41
|
+
},
|
|
42
|
+
"lint-staged": {
|
|
43
|
+
"*.js": "eslint",
|
|
44
|
+
"*.ts": "eslint",
|
|
45
|
+
"*.mjs": "eslint"
|
|
46
|
+
},
|
|
47
|
+
"devDependencies": {
|
|
48
|
+
"@apify/eslint-config": "^1.2.0",
|
|
49
|
+
"@apify/tsconfig": "^0.1.2",
|
|
50
|
+
"@types/node": "^25.9.1",
|
|
51
|
+
"apify-client": "^2.10.0",
|
|
52
|
+
"eslint": "^9.39.4",
|
|
53
|
+
"eslint-config-prettier": "^10.1.8",
|
|
54
|
+
"globals": "^17.7.0",
|
|
55
|
+
"husky": "^9.1.7",
|
|
56
|
+
"knip": "^5.88.1",
|
|
57
|
+
"lint-staged": "^15.5.2",
|
|
58
|
+
"prettier": "^3.9.4",
|
|
59
|
+
"typescript": "^6.0.3",
|
|
60
|
+
"typescript-eslint": "^8.62.1"
|
|
61
|
+
},
|
|
62
|
+
"license": "ISC"
|
|
63
|
+
}
|