@ultimat3/scraping 13.0.0 → 14.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -1
- package/package.json +5 -5
- package/src/browser-record.ts +35 -0
- package/src/cdp-fake.ts +117 -10
- package/src/cdp-port.ts +13 -0
- package/src/cdp-snapshot.ts +37 -6
- package/src/cdp-target.ts +58 -22
- package/src/driver-cdp.ts +4 -0
- package/src/html-target.ts +133 -25
- package/src/http-recorded.ts +9 -2
- package/src/http.ts +26 -3
- package/src/index.ts +6 -1
- package/src/offline-session.ts +3 -0
- package/src/page-over-target.ts +23 -4
- package/src/page.ts +22 -3
- package/src/recording.ts +25 -4
- package/src/rings.ts +20 -0
- package/src/secrets.ts +69 -4
- package/src/session-state.ts +53 -2
- package/src/target.ts +13 -2
package/src/session-state.ts
CHANGED
|
@@ -65,17 +65,68 @@ export const EMPTY_SESSION: SessionSnapshot = Object.freeze({
|
|
|
65
65
|
origin: '',
|
|
66
66
|
});
|
|
67
67
|
|
|
68
|
+
/**
|
|
69
|
+
* 64 bits of hex. A collision here is one account's cookies restored into a run acting as
|
|
70
|
+
* another, so the width is chosen against that and not against convenience: two distinct segments
|
|
71
|
+
* reach a 1% chance of sharing a digest at ~600 million of them. SHA-256 because its output is
|
|
72
|
+
* fixed by its specification — a key minted by one Bun version has to still address the session
|
|
73
|
+
* the previous one wrote, which `Bun.hash`'s families do not promise.
|
|
74
|
+
*/
|
|
75
|
+
const SEGMENT_DIGEST_CHARS = 16;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* One part of the key: readable, path-safe, and INJECTIVE — distinct inputs, distinct segments.
|
|
79
|
+
*
|
|
80
|
+
* The sanitised half is for a human reading a bucket listing. The digest is what makes it a key:
|
|
81
|
+
* `replaceAll(/[^a-zA-Z0-9._-]+/g, '-')` alone COLLAPSES, so `alice@corp.com` and
|
|
82
|
+
* `alice-corp.com` were one segment, `acct/1` and `acct-1` were one segment, and tenants
|
|
83
|
+
* `acme corp` and `acme-corp` were one key space. Every one of those is account A's authenticated
|
|
84
|
+
* session handed to a run acting as account B — `auth.validate()` answers true, because the
|
|
85
|
+
* session IS valid, for the wrong account.
|
|
86
|
+
*
|
|
87
|
+
* Encode rather than collapse, the reasoning `packages/action/src/idempotency-key.ts` states for
|
|
88
|
+
* its JSON tuple: a value that is app data must not be able to spell another value. A JSON tuple
|
|
89
|
+
* cannot be used here because the key is ALSO a storage path — the tenant-first prefix is what an
|
|
90
|
+
* object-store policy scopes — and percent-encoding cannot either, because `%2f` is exactly what
|
|
91
|
+
* `assertSafeKey` refuses. Sanitise for the eye, digest for the identity.
|
|
92
|
+
*
|
|
93
|
+
* Traversal was never what the collapse bought: `assertSafeKey` refuses a `..` segment and still
|
|
94
|
+
* does. The suffix means no segment can BE `..` in the first place.
|
|
95
|
+
*/
|
|
96
|
+
const encodeSegment = (raw: string): string =>
|
|
97
|
+
`${raw.replaceAll(/[^a-zA-Z0-9._-]+/g, '-')}.${new Bun.CryptoHasher('sha256')
|
|
98
|
+
.update(raw)
|
|
99
|
+
.digest('hex')
|
|
100
|
+
.slice(0, SEGMENT_DIGEST_CHARS)}`;
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* The two absent cases, as literals rather than as encoded strings — which is what keeps them
|
|
104
|
+
* unambiguous. Every encoded segment ends in `.` plus 16 hex characters, and neither of these
|
|
105
|
+
* does, so no tenant called `no-tenant` and no `auth.key` returning `default` can land in the
|
|
106
|
+
* key space that means "there was none".
|
|
107
|
+
*/
|
|
108
|
+
const NO_TENANT = 'no-tenant';
|
|
109
|
+
const NO_DISCRIMINATOR = 'default';
|
|
110
|
+
|
|
68
111
|
/**
|
|
69
112
|
* Per tenant, per scrape, per site. The tenant is FIRST because the key is also a storage path,
|
|
70
113
|
* and a prefix that starts with the tenant is one an object-store policy can scope.
|
|
114
|
+
*
|
|
115
|
+
* CHANGES EXISTING KEYS. A session stored under the pre-2026-08-24 spelling is not found under
|
|
116
|
+
* this one, which reads as a cache miss: the run logs in again and writes the new key. No failure,
|
|
117
|
+
* no error, one extra login per stored session — and the old objects are orphaned until the
|
|
118
|
+
* bucket's own lifecycle rule collects them.
|
|
71
119
|
*/
|
|
72
120
|
export function sessionKeyFor(input: {
|
|
73
121
|
readonly scrape: string;
|
|
74
122
|
readonly tenant: string | undefined;
|
|
75
123
|
readonly discriminator?: string | undefined;
|
|
76
124
|
}): string {
|
|
77
|
-
|
|
78
|
-
|
|
125
|
+
return [
|
|
126
|
+
input.tenant === undefined ? NO_TENANT : encodeSegment(input.tenant),
|
|
127
|
+
encodeSegment(input.scrape),
|
|
128
|
+
input.discriminator === undefined ? NO_DISCRIMINATOR : encodeSegment(input.discriminator),
|
|
129
|
+
].join('/');
|
|
79
130
|
}
|
|
80
131
|
|
|
81
132
|
/** What a log line may say about a session: shape, never content. */
|
package/src/target.ts
CHANGED
|
@@ -92,8 +92,10 @@ export interface GotoOptions {
|
|
|
92
92
|
export type CaptureOptions = CaptureFraming;
|
|
93
93
|
|
|
94
94
|
/**
|
|
95
|
-
* The port.
|
|
96
|
-
*
|
|
95
|
+
* The port. Every member is something a browser genuinely does and nothing a scraper's vocabulary
|
|
96
|
+
* should be re-deriving per driver. NO COUNT: this comment said "Twelve methods" while the
|
|
97
|
+
* interface carried seventeen, and a number in prose is wrong the moment the next member lands.
|
|
98
|
+
* `ScrapeTarget` itself is the list.
|
|
97
99
|
*/
|
|
98
100
|
export interface ScrapeTarget {
|
|
99
101
|
/** `puppeteer` | `fixture` | `fake`. Appears in every error cause raised against it. */
|
|
@@ -130,6 +132,15 @@ export interface ScrapeTarget {
|
|
|
130
132
|
select(selector: string, values: readonly string[]): Promise<void>;
|
|
131
133
|
/** The expression runs in the page. The result is `unknown` and is parsed by the caller. */
|
|
132
134
|
evaluate(expression: string): Promise<unknown>;
|
|
135
|
+
/**
|
|
136
|
+
* The browser goes offline, or comes back. REQUIRED on this port where it is optional on
|
|
137
|
+
* `CdpPageLike`, and the asymmetry is the enforcement: a driver author gets a type error naming
|
|
138
|
+
* this member, and the two honest implementations are "set it on the browser" and "refuse by
|
|
139
|
+
* name". A driver with no browser (`fake`, `fixture`) has no network to cut, so it answers
|
|
140
|
+
* `X_NOT_IMPLEMENTED` — never a resolved promise, which would let an offline-behaviour test go
|
|
141
|
+
* green against an app that was online the whole time.
|
|
142
|
+
*/
|
|
143
|
+
setOfflineMode(enabled: boolean): Promise<void>;
|
|
133
144
|
screenshot(options: CaptureOptions): Promise<Uint8Array>;
|
|
134
145
|
pdf(options: CaptureOptions): Promise<Uint8Array>;
|
|
135
146
|
cookies(): Promise<readonly ScrapeCookie[]>;
|