@aplaytest/store-azure 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -0
- package/dist/azure-backend.d.ts +59 -0
- package/dist/azure-backend.d.ts.map +1 -0
- package/dist/azure-backend.js +95 -0
- package/dist/azure-backend.js.map +1 -0
- package/dist/backend.d.ts +30 -0
- package/dist/backend.d.ts.map +1 -0
- package/dist/backend.js +29 -0
- package/dist/backend.js.map +1 -0
- package/dist/blob-store.d.ts +159 -0
- package/dist/blob-store.d.ts.map +1 -0
- package/dist/blob-store.js +282 -0
- package/dist/blob-store.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/layout.d.ts +55 -0
- package/dist/layout.d.ts.map +1 -0
- package/dist/layout.js +115 -0
- package/dist/layout.js.map +1 -0
- package/package.json +40 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The four operations the store needs from object storage.
|
|
3
|
+
*
|
|
4
|
+
* Narrow on purpose. `ContainerClient` has around forty methods, and a store
|
|
5
|
+
* written against it can only be tested against a real account or an emulator
|
|
6
|
+
* — which means in practice it is tested in CI, once, by the pipeline it was
|
|
7
|
+
* supposed to make reliable. Everything interesting about this driver is the
|
|
8
|
+
* naming scheme, the windowing and the merge semantics; none of that is Azure,
|
|
9
|
+
* and all of it is exercised against an in-memory backend in unit tests.
|
|
10
|
+
*/
|
|
11
|
+
export interface BlobBackend {
|
|
12
|
+
/** Blob names under `prefix`, in whatever order the service returns them. */
|
|
13
|
+
list(prefix: string): AsyncIterable<string>;
|
|
14
|
+
/** Blob bytes, or null when it does not exist. Absence is never an error. */
|
|
15
|
+
get(name: string): Promise<Uint8Array | null>;
|
|
16
|
+
put(name: string, body: Uint8Array, contentType: string): Promise<void>;
|
|
17
|
+
remove(name: string): Promise<void>;
|
|
18
|
+
}
|
|
19
|
+
/** In-memory `BlobBackend`, for unit tests and dry-run store construction. */
|
|
20
|
+
export declare class MemoryBlobBackend implements BlobBackend {
|
|
21
|
+
private readonly blobs;
|
|
22
|
+
list(prefix: string): AsyncIterable<string>;
|
|
23
|
+
get(name: string): Promise<Uint8Array | null>;
|
|
24
|
+
put(name: string, body: Uint8Array): Promise<void>;
|
|
25
|
+
remove(name: string): Promise<void>;
|
|
26
|
+
/** Test affordance: how many objects exist, without going through the store. */
|
|
27
|
+
get size(): number;
|
|
28
|
+
names(): string[];
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=backend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../src/backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IAC1B,6EAA6E;IAC7E,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAC5C,6EAA6E;IAC7E,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC9C,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrC;AAED,8EAA8E;AAC9E,qBAAa,iBAAkB,YAAW,WAAW;IACnD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEhD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;IAQ5C,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAI7C,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlD,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzC,gFAAgF;IAChF,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,KAAK,IAAI,MAAM,EAAE;CAGlB"}
|
package/dist/backend.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** In-memory `BlobBackend`, for unit tests and dry-run store construction. */
|
|
2
|
+
export class MemoryBlobBackend {
|
|
3
|
+
blobs = new Map();
|
|
4
|
+
async *list(prefix) {
|
|
5
|
+
// Sorted so a test asserting on ordering is asserting on the store's
|
|
6
|
+
// ordering rather than on Map insertion order.
|
|
7
|
+
for (const name of [...this.blobs.keys()].sort()) {
|
|
8
|
+
if (name.startsWith(prefix))
|
|
9
|
+
yield name;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
async get(name) {
|
|
13
|
+
return this.blobs.get(name) ?? null;
|
|
14
|
+
}
|
|
15
|
+
async put(name, body) {
|
|
16
|
+
this.blobs.set(name, body);
|
|
17
|
+
}
|
|
18
|
+
async remove(name) {
|
|
19
|
+
this.blobs.delete(name);
|
|
20
|
+
}
|
|
21
|
+
/** Test affordance: how many objects exist, without going through the store. */
|
|
22
|
+
get size() {
|
|
23
|
+
return this.blobs.size;
|
|
24
|
+
}
|
|
25
|
+
names() {
|
|
26
|
+
return [...this.blobs.keys()].sort();
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=backend.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backend.js","sourceRoot":"","sources":["../src/backend.ts"],"names":[],"mappings":"AAmBA,8EAA8E;AAC9E,MAAM,OAAO,iBAAiB;IACX,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEvD,KAAK,CAAC,CAAC,IAAI,CAAC,MAAc;QACxB,qEAAqE;QACrE,+CAA+C;QAC/C,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YACjD,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;gBAAE,MAAM,IAAI,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAY,EAAE,IAAgB;QACtC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAY;QACvB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,gFAAgF;IAChF,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,KAAK;QACH,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;CACF"}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `HistoryStore` over an append-only log of run records in Azure Blob Storage.
|
|
3
|
+
*
|
|
4
|
+
* ── Why not the SQLite file that used to live here ──────────────────────────
|
|
5
|
+
* The previous shape downloaded one `history.sqlite`, ingested into it, and
|
|
6
|
+
* re-uploaded it under an `If-Match` precondition. It was small and it worked
|
|
7
|
+
* for one writer. What it could not do:
|
|
8
|
+
*
|
|
9
|
+
* · TOLERATE OVERLAP. Two main-branch runs finishing together meant one lost
|
|
10
|
+
* the conditional PUT. Failing is the good outcome; the workflow that
|
|
11
|
+
* retried without re-downloading silently discarded the other run.
|
|
12
|
+
* · STAY SMALL. Every run rewrote the whole database to append one record.
|
|
13
|
+
* · WINDOW. Scoring reads 90 days; the file was all of history or nothing.
|
|
14
|
+
*
|
|
15
|
+
* Objects fix all three by construction — see `layout.ts` for how the name
|
|
16
|
+
* does the work. What is left is this: fetch a window, hand it to the shared
|
|
17
|
+
* `HistoryIndex`, and let the query semantics be identical to the SQL driver's
|
|
18
|
+
* because they are literally the same code.
|
|
19
|
+
*
|
|
20
|
+
* ── The read pattern this is shaped for ─────────────────────────────────────
|
|
21
|
+
* `analyzeAll` calls `attempts()` twice per (test, project) — a few hundred
|
|
22
|
+
* calls for a real suite. A store that went to the network per call would take
|
|
23
|
+
* minutes and cost a fortune in transactions. So the window is downloaded ONCE,
|
|
24
|
+
* lazily, on the first read, and every query after that is served from memory.
|
|
25
|
+
* Writes do not trigger a load: ingesting a shard is one PUT.
|
|
26
|
+
*/
|
|
27
|
+
import { type AzureBlobTarget, type HistoricalAttempt, type HistoryQuery, type HistoryStore, type RunRecord, type TestKey } from '@aplaytest/core';
|
|
28
|
+
import type { BlobBackend } from './backend.js';
|
|
29
|
+
/** Default read window. Matches `history.retainDays`, so reads see what prune keeps. */
|
|
30
|
+
export declare const DEFAULT_WINDOW_DAYS = 90;
|
|
31
|
+
/**
|
|
32
|
+
* Parallel downloads. Blob GETs are independent and mostly latency, so this is
|
|
33
|
+
* the difference between a 4-second load and a 90-second one; past ~32 the
|
|
34
|
+
* account starts throttling and the retries give the time back.
|
|
35
|
+
*/
|
|
36
|
+
export declare const DEFAULT_CONCURRENCY = 16;
|
|
37
|
+
export interface BlobHistoryStoreOptions {
|
|
38
|
+
/** Key prefix inside the container. Normalised to end with `/`, or empty. */
|
|
39
|
+
readonly prefix?: string;
|
|
40
|
+
/** Days of history to read. `null` reads everything — slower every week. */
|
|
41
|
+
readonly windowDays?: number | null;
|
|
42
|
+
readonly concurrency?: number;
|
|
43
|
+
/** Injectable clock, so the window boundary is testable. */
|
|
44
|
+
readonly now?: () => number;
|
|
45
|
+
/**
|
|
46
|
+
* Score against the store; never write to it. See `AzureBlobTarget.readOnly`
|
|
47
|
+
* for why this is a mode rather than a permission error.
|
|
48
|
+
*
|
|
49
|
+
* `ingest()` still indexes the record locally, so the run being analysed is
|
|
50
|
+
* scored alongside the downloaded baseline — the PR sees its own results in
|
|
51
|
+
* context, it just does not leave them behind.
|
|
52
|
+
*/
|
|
53
|
+
readonly readOnly?: boolean;
|
|
54
|
+
}
|
|
55
|
+
/** A blob a read could not use. Reported, never thrown — see `skipped`. */
|
|
56
|
+
export interface SkippedBlob {
|
|
57
|
+
readonly name: string;
|
|
58
|
+
readonly reason: string;
|
|
59
|
+
}
|
|
60
|
+
export declare class BlobHistoryStore implements HistoryStore {
|
|
61
|
+
private readonly backend;
|
|
62
|
+
private readonly index;
|
|
63
|
+
private readonly prefix;
|
|
64
|
+
private readonly windowDays;
|
|
65
|
+
private readonly concurrency;
|
|
66
|
+
private readonly now;
|
|
67
|
+
private readonly readOnly;
|
|
68
|
+
/** Listing, once fetched. Answers `runCount` and `prune` with no downloads. */
|
|
69
|
+
private entries;
|
|
70
|
+
/** Whether the window's records are materialised in the index. */
|
|
71
|
+
private materialised;
|
|
72
|
+
private readonly skippedBlobs;
|
|
73
|
+
constructor(backend: BlobBackend, options?: BlobHistoryStoreOptions);
|
|
74
|
+
/**
|
|
75
|
+
* Blobs that exist but could not be read: truncated uploads, records written
|
|
76
|
+
* by a newer atest, anything hand-edited into the container.
|
|
77
|
+
*
|
|
78
|
+
* Surfaced rather than thrown, for the same reason `ingestDirectory` skips a
|
|
79
|
+
* bad file instead of failing the batch — one unreadable object among two
|
|
80
|
+
* thousand must not cost you the other 1,999. The CLI prints these, because
|
|
81
|
+
* a store that silently reads less than it should looks exactly like a store
|
|
82
|
+
* with nothing in it.
|
|
83
|
+
*/
|
|
84
|
+
get skipped(): readonly SkippedBlob[];
|
|
85
|
+
/** `YYYY-MM-DD` before which blobs are outside the read window. */
|
|
86
|
+
private windowStart;
|
|
87
|
+
private ensureListing;
|
|
88
|
+
private ensureMaterialised;
|
|
89
|
+
/**
|
|
90
|
+
* One PUT. No read, no lock, no precondition — the name is derived from the
|
|
91
|
+
* run id and the shard, so a re-ingest overwrites exactly itself and two
|
|
92
|
+
* concurrent writers cannot collide.
|
|
93
|
+
*/
|
|
94
|
+
ingest(run: RunRecord): Promise<void>;
|
|
95
|
+
attempts(query?: HistoryQuery): Promise<HistoricalAttempt[]>;
|
|
96
|
+
testKeys(): Promise<TestKey[]>;
|
|
97
|
+
/** Answered from the listing alone: the run id is in the blob name. */
|
|
98
|
+
runCount(): Promise<number>;
|
|
99
|
+
/**
|
|
100
|
+
* Delete runs that started before the cutoff.
|
|
101
|
+
*
|
|
102
|
+
* DAY GRANULARITY. Blobs are partitioned by date, so this deletes whole days
|
|
103
|
+
* strictly older than the cutoff's date and leaves the cutoff day alone. A
|
|
104
|
+
* retention window is a rolling approximation — trimming to the hour would
|
|
105
|
+
* mean downloading every record on the boundary day to read a timestamp
|
|
106
|
+
* already encoded, less precisely, in its name.
|
|
107
|
+
*
|
|
108
|
+
* Consider leaving retention to the account's lifecycle-management policy
|
|
109
|
+
* instead; terraform sets one up. This exists for the case where you want it
|
|
110
|
+
* to happen now, and for stores without a policy.
|
|
111
|
+
*
|
|
112
|
+
* One known edge: a sharded run that crosses midnight has shards in two
|
|
113
|
+
* partitions, and a cutoff landing between them prunes half of it. The run
|
|
114
|
+
* is at the far edge of the window and about to age out entirely, so the
|
|
115
|
+
* alternative — reading every record on the boundary day to recover a
|
|
116
|
+
* timestamp already in the name — buys a day of precision for a download.
|
|
117
|
+
*/
|
|
118
|
+
prune(olderThanIso: string): Promise<number>;
|
|
119
|
+
close(): Promise<void>;
|
|
120
|
+
}
|
|
121
|
+
/** Build a store from a parsed `azblob://` URL. The path the CLI takes. */
|
|
122
|
+
export interface OpenBlobHistoryStoreOptions extends Omit<BlobHistoryStoreOptions, 'prefix' | 'windowDays' | 'readOnly'> {
|
|
123
|
+
/**
|
|
124
|
+
* Shared-key credential, for an EMULATOR. Unset — the normal case — the
|
|
125
|
+
* driver uses `DefaultAzureCredential` and authenticates with Entra.
|
|
126
|
+
*
|
|
127
|
+
* This exists so the documented Azurite URL form is actually usable: the
|
|
128
|
+
* emulator has no Entra to federate against, so without a key the CLI can
|
|
129
|
+
* parse `http://127.0.0.1:10000/devstoreaccount1/…` and then fail to
|
|
130
|
+
* authenticate against it — a supported-looking target that never works.
|
|
131
|
+
*
|
|
132
|
+
* It does NOT weaken production. Keys are refused by the STORAGE ACCOUNT
|
|
133
|
+
* (`shared_access_key_enabled = false`), so what a client is willing to
|
|
134
|
+
* construct is irrelevant there; a key passed at a real account fails at the
|
|
135
|
+
* service. The control lives in Azure, not in this signature.
|
|
136
|
+
*/
|
|
137
|
+
readonly accountKey?: string | undefined;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Build a {@link BlobHistoryStore} from a parsed `azblob://` URL.
|
|
141
|
+
*
|
|
142
|
+
* This is the path the CLI takes. Authentication is `DefaultAzureCredential`
|
|
143
|
+
* unless `accountKey` is set, which exists only so the Azurite emulator URL
|
|
144
|
+
* is actually usable (the emulator has no Entra to federate against).
|
|
145
|
+
*
|
|
146
|
+
* @param target - Parsed Azure Blob target (`parseHistoryUrl` of an `azblob://` URL).
|
|
147
|
+
* @param options - Optional emulator shared key and concurrency.
|
|
148
|
+
* @returns A store that implements `HistoryStore`.
|
|
149
|
+
*
|
|
150
|
+
* @example
|
|
151
|
+
* ```ts
|
|
152
|
+
* const target = parseHistoryUrl('azblob://myaccount/atest-history?readonly=1');
|
|
153
|
+
* if (target.kind !== 'azure-blob') throw new Error('expected azure-blob');
|
|
154
|
+
* const store = openBlobHistoryStore(target);
|
|
155
|
+
* const attempts = await store.attempts();
|
|
156
|
+
* ```
|
|
157
|
+
*/
|
|
158
|
+
export declare function openBlobHistoryStore(target: AzureBlobTarget, options?: OpenBlobHistoryStoreOptions): BlobHistoryStore;
|
|
159
|
+
//# sourceMappingURL=blob-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"blob-store.d.ts","sourceRoot":"","sources":["../src/blob-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAKH,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,SAAS,EACd,KAAK,OAAO,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAShD,wFAAwF;AACxF,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,MAAM,WAAW,uBAAuB;IACtC,6EAA6E;IAC7E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,4EAA4E;IAC5E,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,4DAA4D;IAC5D,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,2EAA2E;AAC3E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AA2CD,qBAAa,gBAAiB,YAAW,YAAY;IAgBjD,OAAO,CAAC,QAAQ,CAAC,OAAO;IAf1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgB;IAC3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IAEnC,+EAA+E;IAC/E,OAAO,CAAC,OAAO,CAAwB;IACvC,kEAAkE;IAClE,OAAO,CAAC,YAAY,CAAS;IAE7B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAqB;gBAG/B,OAAO,EAAE,WAAW,EACrC,OAAO,GAAE,uBAA4B;IASvC;;;;;;;;;OASG;IACH,IAAI,OAAO,IAAI,SAAS,WAAW,EAAE,CAEpC;IAED,mEAAmE;IACnE,OAAO,CAAC,WAAW;YAKL,aAAa;YAoBb,kBAAkB;IAqChC;;;;OAIG;IACG,MAAM,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBrC,QAAQ,CAAC,KAAK,GAAE,YAAiB,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAKhE,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAKpC,uEAAuE;IACjE,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;IAKjC;;;;;;;;;;;;;;;;;;OAkBG;IACG,KAAK,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAsB5C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B;AAED,2EAA2E;AAC3E,MAAM,WAAW,2BACf,SAAQ,IAAI,CAAC,uBAAuB,EAAE,QAAQ,GAAG,YAAY,GAAG,UAAU,CAAC;IAC3E;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,eAAe,EACvB,OAAO,GAAE,2BAAgC,GACxC,gBAAgB,CAelB"}
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `HistoryStore` over an append-only log of run records in Azure Blob Storage.
|
|
3
|
+
*
|
|
4
|
+
* ── Why not the SQLite file that used to live here ──────────────────────────
|
|
5
|
+
* The previous shape downloaded one `history.sqlite`, ingested into it, and
|
|
6
|
+
* re-uploaded it under an `If-Match` precondition. It was small and it worked
|
|
7
|
+
* for one writer. What it could not do:
|
|
8
|
+
*
|
|
9
|
+
* · TOLERATE OVERLAP. Two main-branch runs finishing together meant one lost
|
|
10
|
+
* the conditional PUT. Failing is the good outcome; the workflow that
|
|
11
|
+
* retried without re-downloading silently discarded the other run.
|
|
12
|
+
* · STAY SMALL. Every run rewrote the whole database to append one record.
|
|
13
|
+
* · WINDOW. Scoring reads 90 days; the file was all of history or nothing.
|
|
14
|
+
*
|
|
15
|
+
* Objects fix all three by construction — see `layout.ts` for how the name
|
|
16
|
+
* does the work. What is left is this: fetch a window, hand it to the shared
|
|
17
|
+
* `HistoryIndex`, and let the query semantics be identical to the SQL driver's
|
|
18
|
+
* because they are literally the same code.
|
|
19
|
+
*
|
|
20
|
+
* ── The read pattern this is shaped for ─────────────────────────────────────
|
|
21
|
+
* `analyzeAll` calls `attempts()` twice per (test, project) — a few hundred
|
|
22
|
+
* calls for a real suite. A store that went to the network per call would take
|
|
23
|
+
* minutes and cost a fortune in transactions. So the window is downloaded ONCE,
|
|
24
|
+
* lazily, on the first read, and every query after that is served from memory.
|
|
25
|
+
* Writes do not trigger a load: ingesting a shard is one PUT.
|
|
26
|
+
*/
|
|
27
|
+
import { gunzip, gzip } from 'node:zlib';
|
|
28
|
+
import { promisify } from 'node:util';
|
|
29
|
+
import { HistoryIndex, RUN_SCHEMA_VERSION, shardKeyOf, } from '@aplaytest/core';
|
|
30
|
+
import { StorageSharedKeyCredential } from '@azure/storage-blob';
|
|
31
|
+
import { AzureBlobBackend } from './azure-backend.js';
|
|
32
|
+
import { parseRunBlobName, runBlobName, runsPrefix } from './layout.js';
|
|
33
|
+
const gzipAsync = promisify(gzip);
|
|
34
|
+
const gunzipAsync = promisify(gunzip);
|
|
35
|
+
/** Default read window. Matches `history.retainDays`, so reads see what prune keeps. */
|
|
36
|
+
export const DEFAULT_WINDOW_DAYS = 90;
|
|
37
|
+
/**
|
|
38
|
+
* Parallel downloads. Blob GETs are independent and mostly latency, so this is
|
|
39
|
+
* the difference between a 4-second load and a 90-second one; past ~32 the
|
|
40
|
+
* account starts throttling and the retries give the time back.
|
|
41
|
+
*/
|
|
42
|
+
export const DEFAULT_CONCURRENCY = 16;
|
|
43
|
+
const GZIP_MAGIC = [0x1f, 0x8b];
|
|
44
|
+
function isGzip(bytes) {
|
|
45
|
+
return bytes[0] === GZIP_MAGIC[0] && bytes[1] === GZIP_MAGIC[1];
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Run a bounded number of tasks at once, preserving nothing about order.
|
|
49
|
+
*
|
|
50
|
+
* `Promise.all` over 2,000 blobs opens 2,000 sockets and gets the account to
|
|
51
|
+
* throttle; a sequential loop turns a 4-second load into a 20-minute one.
|
|
52
|
+
*/
|
|
53
|
+
async function pool(items, limit, work) {
|
|
54
|
+
let cursor = 0;
|
|
55
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
56
|
+
for (;;) {
|
|
57
|
+
const index = cursor;
|
|
58
|
+
cursor += 1;
|
|
59
|
+
const item = items[index];
|
|
60
|
+
if (item === undefined)
|
|
61
|
+
return;
|
|
62
|
+
await work(item);
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
await Promise.all(workers);
|
|
66
|
+
}
|
|
67
|
+
function isRunRecord(value) {
|
|
68
|
+
if (typeof value !== 'object' || value === null)
|
|
69
|
+
return false;
|
|
70
|
+
const record = value;
|
|
71
|
+
return (record['schemaVersion'] === RUN_SCHEMA_VERSION &&
|
|
72
|
+
typeof record['runId'] === 'string' &&
|
|
73
|
+
record['runId'] !== '' &&
|
|
74
|
+
Array.isArray(record['attempts']));
|
|
75
|
+
}
|
|
76
|
+
export class BlobHistoryStore {
|
|
77
|
+
backend;
|
|
78
|
+
index = new HistoryIndex();
|
|
79
|
+
prefix;
|
|
80
|
+
windowDays;
|
|
81
|
+
concurrency;
|
|
82
|
+
now;
|
|
83
|
+
readOnly;
|
|
84
|
+
/** Listing, once fetched. Answers `runCount` and `prune` with no downloads. */
|
|
85
|
+
entries = null;
|
|
86
|
+
/** Whether the window's records are materialised in the index. */
|
|
87
|
+
materialised = false;
|
|
88
|
+
skippedBlobs = [];
|
|
89
|
+
constructor(backend, options = {}) {
|
|
90
|
+
this.backend = backend;
|
|
91
|
+
this.prefix = options.prefix ?? '';
|
|
92
|
+
this.windowDays = options.windowDays === undefined ? DEFAULT_WINDOW_DAYS : options.windowDays;
|
|
93
|
+
this.concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
|
|
94
|
+
this.now = options.now ?? Date.now;
|
|
95
|
+
this.readOnly = options.readOnly ?? false;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Blobs that exist but could not be read: truncated uploads, records written
|
|
99
|
+
* by a newer atest, anything hand-edited into the container.
|
|
100
|
+
*
|
|
101
|
+
* Surfaced rather than thrown, for the same reason `ingestDirectory` skips a
|
|
102
|
+
* bad file instead of failing the batch — one unreadable object among two
|
|
103
|
+
* thousand must not cost you the other 1,999. The CLI prints these, because
|
|
104
|
+
* a store that silently reads less than it should looks exactly like a store
|
|
105
|
+
* with nothing in it.
|
|
106
|
+
*/
|
|
107
|
+
get skipped() {
|
|
108
|
+
return this.skippedBlobs;
|
|
109
|
+
}
|
|
110
|
+
/** `YYYY-MM-DD` before which blobs are outside the read window. */
|
|
111
|
+
windowStart() {
|
|
112
|
+
if (this.windowDays === null)
|
|
113
|
+
return null;
|
|
114
|
+
return new Date(this.now() - this.windowDays * 86_400_000).toISOString().slice(0, 10);
|
|
115
|
+
}
|
|
116
|
+
async ensureListing() {
|
|
117
|
+
if (this.entries !== null)
|
|
118
|
+
return this.entries;
|
|
119
|
+
const start = this.windowStart();
|
|
120
|
+
const found = [];
|
|
121
|
+
for await (const name of this.backend.list(runsPrefix(this.prefix))) {
|
|
122
|
+
const parsed = parseRunBlobName(this.prefix, name);
|
|
123
|
+
if (parsed === null)
|
|
124
|
+
continue;
|
|
125
|
+
if (start !== null && parsed.date < start)
|
|
126
|
+
continue;
|
|
127
|
+
found.push({ ...parsed, name });
|
|
128
|
+
}
|
|
129
|
+
// Sorted by name, which sorts by date then run id then shard. Metadata
|
|
130
|
+
// merging is last-writer-wins, so a stable order is the difference between
|
|
131
|
+
// a reproducible report and one that changes with the listing's paging.
|
|
132
|
+
found.sort((a, b) => a.name.localeCompare(b.name));
|
|
133
|
+
this.entries = found;
|
|
134
|
+
return found;
|
|
135
|
+
}
|
|
136
|
+
async ensureMaterialised() {
|
|
137
|
+
if (this.materialised)
|
|
138
|
+
return;
|
|
139
|
+
const entries = await this.ensureListing();
|
|
140
|
+
const pending = entries.filter(entry => !this.index.has(entry.runId, entry.shardKey));
|
|
141
|
+
await pool(pending, this.concurrency, async (entry) => {
|
|
142
|
+
const bytes = await this.backend.get(entry.name);
|
|
143
|
+
// Pruned between the listing and the download. Not an error.
|
|
144
|
+
if (bytes === null)
|
|
145
|
+
return;
|
|
146
|
+
let record;
|
|
147
|
+
try {
|
|
148
|
+
const json = isGzip(bytes)
|
|
149
|
+
? (await gunzipAsync(bytes)).toString('utf8')
|
|
150
|
+
: new TextDecoder().decode(bytes);
|
|
151
|
+
record = JSON.parse(json);
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
this.skippedBlobs.push({
|
|
155
|
+
name: entry.name,
|
|
156
|
+
reason: error instanceof Error ? error.message : 'unreadable',
|
|
157
|
+
});
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
if (!isRunRecord(record)) {
|
|
161
|
+
this.skippedBlobs.push({
|
|
162
|
+
name: entry.name,
|
|
163
|
+
reason: `not a schemaVersion ${RUN_SCHEMA_VERSION} run record`,
|
|
164
|
+
});
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
this.index.add(record);
|
|
168
|
+
});
|
|
169
|
+
this.materialised = true;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* One PUT. No read, no lock, no precondition — the name is derived from the
|
|
173
|
+
* run id and the shard, so a re-ingest overwrites exactly itself and two
|
|
174
|
+
* concurrent writers cannot collide.
|
|
175
|
+
*/
|
|
176
|
+
async ingest(run) {
|
|
177
|
+
const shardKey = shardKeyOf(run.shard);
|
|
178
|
+
const name = runBlobName(this.prefix, run.startedAt, run.runId, shardKey);
|
|
179
|
+
// Indexed either way: a pull request scores its own run against the trunk
|
|
180
|
+
// baseline. Read-only decides whether the record outlives the job.
|
|
181
|
+
this.index.add(run);
|
|
182
|
+
if (this.readOnly)
|
|
183
|
+
return;
|
|
184
|
+
const body = await gzipAsync(Buffer.from(JSON.stringify(run), 'utf8'));
|
|
185
|
+
await this.backend.put(name, new Uint8Array(body), 'application/json');
|
|
186
|
+
// Keep an already-fetched listing honest, so `runCount()` right after an
|
|
187
|
+
// ingest does not have to re-list to see what this process just wrote.
|
|
188
|
+
if (this.entries !== null && !this.entries.some(entry => entry.name === name)) {
|
|
189
|
+
const parsed = parseRunBlobName(this.prefix, name);
|
|
190
|
+
if (parsed !== null)
|
|
191
|
+
this.entries.push({ ...parsed, name });
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
async attempts(query = {}) {
|
|
195
|
+
await this.ensureMaterialised();
|
|
196
|
+
return this.index.attempts(query);
|
|
197
|
+
}
|
|
198
|
+
async testKeys() {
|
|
199
|
+
await this.ensureMaterialised();
|
|
200
|
+
return this.index.testKeys();
|
|
201
|
+
}
|
|
202
|
+
/** Answered from the listing alone: the run id is in the blob name. */
|
|
203
|
+
async runCount() {
|
|
204
|
+
const entries = await this.ensureListing();
|
|
205
|
+
return new Set(entries.map(entry => entry.runId)).size;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Delete runs that started before the cutoff.
|
|
209
|
+
*
|
|
210
|
+
* DAY GRANULARITY. Blobs are partitioned by date, so this deletes whole days
|
|
211
|
+
* strictly older than the cutoff's date and leaves the cutoff day alone. A
|
|
212
|
+
* retention window is a rolling approximation — trimming to the hour would
|
|
213
|
+
* mean downloading every record on the boundary day to read a timestamp
|
|
214
|
+
* already encoded, less precisely, in its name.
|
|
215
|
+
*
|
|
216
|
+
* Consider leaving retention to the account's lifecycle-management policy
|
|
217
|
+
* instead; terraform sets one up. This exists for the case where you want it
|
|
218
|
+
* to happen now, and for stores without a policy.
|
|
219
|
+
*
|
|
220
|
+
* One known edge: a sharded run that crosses midnight has shards in two
|
|
221
|
+
* partitions, and a cutoff landing between them prunes half of it. The run
|
|
222
|
+
* is at the far edge of the window and about to age out entirely, so the
|
|
223
|
+
* alternative — reading every record on the boundary day to recover a
|
|
224
|
+
* timestamp already in the name — buys a day of precision for a download.
|
|
225
|
+
*/
|
|
226
|
+
async prune(olderThanIso) {
|
|
227
|
+
if (this.readOnly) {
|
|
228
|
+
throw new Error('This store is open read-only, so prune would delete nothing and report success.\n' +
|
|
229
|
+
' Retention is a main-branch operation: drop ?readonly=1 from the URL, and expect\n' +
|
|
230
|
+
' the identity to hold "Storage Blob Data Contributor".');
|
|
231
|
+
}
|
|
232
|
+
const entries = await this.ensureListing();
|
|
233
|
+
const cutoff = olderThanIso.slice(0, 10);
|
|
234
|
+
const doomed = entries.filter(entry => entry.date < cutoff);
|
|
235
|
+
await pool(doomed, this.concurrency, entry => this.backend.remove(entry.name));
|
|
236
|
+
const removedNames = new Set(doomed.map(entry => entry.name));
|
|
237
|
+
this.entries = entries.filter(entry => !removedNames.has(entry.name));
|
|
238
|
+
this.index.prune(`${cutoff}T00:00:00.000Z`);
|
|
239
|
+
return new Set(doomed.map(entry => entry.runId)).size;
|
|
240
|
+
}
|
|
241
|
+
async close() {
|
|
242
|
+
this.index.clear();
|
|
243
|
+
this.entries = null;
|
|
244
|
+
this.materialised = false;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Build a {@link BlobHistoryStore} from a parsed `azblob://` URL.
|
|
249
|
+
*
|
|
250
|
+
* This is the path the CLI takes. Authentication is `DefaultAzureCredential`
|
|
251
|
+
* unless `accountKey` is set, which exists only so the Azurite emulator URL
|
|
252
|
+
* is actually usable (the emulator has no Entra to federate against).
|
|
253
|
+
*
|
|
254
|
+
* @param target - Parsed Azure Blob target (`parseHistoryUrl` of an `azblob://` URL).
|
|
255
|
+
* @param options - Optional emulator shared key and concurrency.
|
|
256
|
+
* @returns A store that implements `HistoryStore`.
|
|
257
|
+
*
|
|
258
|
+
* @example
|
|
259
|
+
* ```ts
|
|
260
|
+
* const target = parseHistoryUrl('azblob://myaccount/atest-history?readonly=1');
|
|
261
|
+
* if (target.kind !== 'azure-blob') throw new Error('expected azure-blob');
|
|
262
|
+
* const store = openBlobHistoryStore(target);
|
|
263
|
+
* const attempts = await store.attempts();
|
|
264
|
+
* ```
|
|
265
|
+
*/
|
|
266
|
+
export function openBlobHistoryStore(target, options = {}) {
|
|
267
|
+
const backend = new AzureBlobBackend({
|
|
268
|
+
serviceUrl: target.serviceUrl,
|
|
269
|
+
container: target.container,
|
|
270
|
+
...(options.accountKey === undefined
|
|
271
|
+
? {}
|
|
272
|
+
: { credential: new StorageSharedKeyCredential(target.account, options.accountKey) }),
|
|
273
|
+
});
|
|
274
|
+
return new BlobHistoryStore(backend, {
|
|
275
|
+
...options,
|
|
276
|
+
prefix: target.prefix,
|
|
277
|
+
// `?window=` on the URL wins; unset means the default, never "everything".
|
|
278
|
+
windowDays: target.windowDays ?? DEFAULT_WINDOW_DAYS,
|
|
279
|
+
readOnly: target.readOnly,
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
//# sourceMappingURL=blob-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"blob-store.js","sourceRoot":"","sources":["../src/blob-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,UAAU,GAOX,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAoB,MAAM,aAAa,CAAC;AAE1F,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;AAClC,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;AAEtC,wFAAwF;AACxF,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAEtC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AA+BtC,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AAEhC,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,IAAI,CAAI,KAAmB,EAAE,KAAa,EAAE,IAAgC;IACzF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC/E,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,MAAM,CAAC;YACrB,MAAM,IAAI,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,OAAO;YAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,OAAO,CACL,MAAM,CAAC,eAAe,CAAC,KAAK,kBAAkB;QAC9C,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,QAAQ;QACnC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE;QACtB,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAClC,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,gBAAgB;IAgBR;IAfF,KAAK,GAAG,IAAI,YAAY,EAAE,CAAC;IAC3B,MAAM,CAAS;IACf,UAAU,CAAgB;IAC1B,WAAW,CAAS;IACpB,GAAG,CAAe;IAClB,QAAQ,CAAU;IAEnC,+EAA+E;IACvE,OAAO,GAAmB,IAAI,CAAC;IACvC,kEAAkE;IAC1D,YAAY,GAAG,KAAK,CAAC;IAEZ,YAAY,GAAkB,EAAE,CAAC;IAElD,YACmB,OAAoB,EACrC,UAAmC,EAAE;QADpB,YAAO,GAAP,OAAO,CAAa;QAGrC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QAC9F,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,mBAAmB,CAAC;QAC9D,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC5C,CAAC;IAED;;;;;;;;;OASG;IACH,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,mEAAmE;IAC3D,WAAW;QACjB,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC1C,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxF,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC;QAE/C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,KAAK,GAAY,EAAE,CAAC;QAC1B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACnD,IAAI,MAAM,KAAK,IAAI;gBAAE,SAAS;YAC9B,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,IAAI,GAAG,KAAK;gBAAE,SAAS;YACpD,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,uEAAuE;QACvE,2EAA2E;QAC3E,wEAAwE;QACxE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QAC9B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAE3C,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAC,KAAK,EAAC,EAAE;YAClD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACjD,6DAA6D;YAC7D,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO;YAE3B,IAAI,MAAe,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC;oBACxB,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC7C,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;oBACrB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY;iBAC9D,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;oBACrB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,MAAM,EAAE,uBAAuB,kBAAkB,aAAa;iBAC/D,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,CAAC,GAAc;QACzB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE1E,0EAA0E;QAC1E,mEAAmE;QACnE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpB,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE1B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACvE,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAEvE,yEAAyE;QACzE,uEAAuE;QACvE,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YAC9E,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACnD,IAAI,MAAM,KAAK,IAAI;gBAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,QAAsB,EAAE;QACrC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,QAAQ;QACZ,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3C,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,KAAK,CAAC,YAAoB;QAC9B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,mFAAmF;gBACjF,qFAAqF;gBACrF,yDAAyD,CAC5D,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAE/E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACtE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,MAAM,gBAAgB,CAAC,CAAC;QAE5C,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;CACF;AAsBD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,oBAAoB,CAClC,MAAuB,EACvB,UAAuC,EAAE;IAEzC,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;QACnC,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS;YAClC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,0BAA0B,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;KACxF,CAAC,CAAC;IACH,OAAO,IAAI,gBAAgB,CAAC,OAAO,EAAE;QACnC,GAAG,OAAO;QACV,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,2EAA2E;QAC3E,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,mBAAmB;QACpD,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;AACL,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @aplaytest/store-azure — run history in Azure Blob Storage.
|
|
3
|
+
*
|
|
4
|
+
* Split out of `@aplaytest/core` deliberately. Core's contract is "no Playwright,
|
|
5
|
+
* no network, no model", and it is the package every consumer installs,
|
|
6
|
+
* including the reporter that loads inside the test process. Putting an 8 MB
|
|
7
|
+
* SDK and a credential chain behind that import would make every Playwright
|
|
8
|
+
* run pay for a feature only the analyze job uses.
|
|
9
|
+
*
|
|
10
|
+
* Install it alongside the CLI where history has to outlive the run:
|
|
11
|
+
*
|
|
12
|
+
* npm i -D @aplaytest/cli @aplaytest/store-azure
|
|
13
|
+
* aplaytest history ingest --db azblob://<account>/atest-history --runs .atest/runs
|
|
14
|
+
*/
|
|
15
|
+
export { BlobHistoryStore, openBlobHistoryStore, DEFAULT_WINDOW_DAYS, DEFAULT_CONCURRENCY } from './blob-store.js';
|
|
16
|
+
export type { BlobHistoryStoreOptions, OpenBlobHistoryStoreOptions, SkippedBlob, } from './blob-store.js';
|
|
17
|
+
export { AzureBlobBackend } from './azure-backend.js';
|
|
18
|
+
export type { AzureBlobBackendOptions } from './azure-backend.js';
|
|
19
|
+
export { MemoryBlobBackend } from './backend.js';
|
|
20
|
+
export type { BlobBackend } from './backend.js';
|
|
21
|
+
export { LAYOUT_VERSION, LayoutError, encodeSegment, decodeSegment, partitionOf, runBlobName, parseRunBlobName, runsPrefix, } from './layout.js';
|
|
22
|
+
export type { RunBlobName } from './layout.js';
|
|
23
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACnH,YAAY,EACV,uBAAuB,EACvB,2BAA2B,EAC3B,WAAW,GACZ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,YAAY,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAElE,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,OAAO,EACL,cAAc,EACd,WAAW,EACX,aAAa,EACb,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,UAAU,GACX,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @aplaytest/store-azure — run history in Azure Blob Storage.
|
|
3
|
+
*
|
|
4
|
+
* Split out of `@aplaytest/core` deliberately. Core's contract is "no Playwright,
|
|
5
|
+
* no network, no model", and it is the package every consumer installs,
|
|
6
|
+
* including the reporter that loads inside the test process. Putting an 8 MB
|
|
7
|
+
* SDK and a credential chain behind that import would make every Playwright
|
|
8
|
+
* run pay for a feature only the analyze job uses.
|
|
9
|
+
*
|
|
10
|
+
* Install it alongside the CLI where history has to outlive the run:
|
|
11
|
+
*
|
|
12
|
+
* npm i -D @aplaytest/cli @aplaytest/store-azure
|
|
13
|
+
* aplaytest history ingest --db azblob://<account>/atest-history --runs .atest/runs
|
|
14
|
+
*/
|
|
15
|
+
export { BlobHistoryStore, openBlobHistoryStore, DEFAULT_WINDOW_DAYS, DEFAULT_CONCURRENCY } from './blob-store.js';
|
|
16
|
+
export { AzureBlobBackend } from './azure-backend.js';
|
|
17
|
+
export { MemoryBlobBackend } from './backend.js';
|
|
18
|
+
export { LAYOUT_VERSION, LayoutError, encodeSegment, decodeSegment, partitionOf, runBlobName, parseRunBlobName, runsPrefix, } from './layout.js';
|
|
19
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAOnH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAGtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAGjD,OAAO,EACL,cAAc,EACd,WAAW,EACX,aAAa,EACb,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,UAAU,GACX,MAAM,aAAa,CAAC"}
|
package/dist/layout.d.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The blob naming scheme — the whole concurrency story, encoded in a path.
|
|
3
|
+
*
|
|
4
|
+
* The store this replaces was one SQLite file on a blob: download it, ingest,
|
|
5
|
+
* upload it back under an ETag precondition. That works exactly until two runs
|
|
6
|
+
* overlap, at which point the loser either fails the step or discards the
|
|
7
|
+
* winner's attempts, and it re-uploads the entire history to record thirty
|
|
8
|
+
* seconds of it.
|
|
9
|
+
*
|
|
10
|
+
* Here a run record is an object and the container is an append-only log:
|
|
11
|
+
*
|
|
12
|
+
* <prefix>v1/runs/2026/08/30/<runId>/<shard>.json.gz
|
|
13
|
+
*
|
|
14
|
+
* Three properties follow directly from the name, with no locking:
|
|
15
|
+
*
|
|
16
|
+
* · IDEMPOTENT. A re-ingested shard computes the same name and overwrites
|
|
17
|
+
* itself. The scoped-delete and UPSERT gymnastics the SQL driver needs to
|
|
18
|
+
* avoid double-counting are a naming rule here.
|
|
19
|
+
* · CONCURRENT. Two shards of one run, and two overlapping main-branch runs,
|
|
20
|
+
* write different names. Nothing races, so nothing needs a precondition.
|
|
21
|
+
* · CHEAP TO WINDOW. The date is in the path, so a 90-day read filters the
|
|
22
|
+
* LISTING and downloads only what it will score, and the run id is in the
|
|
23
|
+
* path, so counting runs needs no downloads at all.
|
|
24
|
+
*
|
|
25
|
+
* All pure string work: no client, no credential, no network.
|
|
26
|
+
*/
|
|
27
|
+
/** Bumped only for a change no existing reader can parse. Old prefixes stay readable. */
|
|
28
|
+
export declare const LAYOUT_VERSION = "v1";
|
|
29
|
+
export declare function encodeSegment(value: string): string;
|
|
30
|
+
export declare function decodeSegment(value: string): string;
|
|
31
|
+
export declare class LayoutError extends Error {
|
|
32
|
+
constructor(message: string);
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The date partition a run belongs to, taken from `startedAt`.
|
|
36
|
+
*
|
|
37
|
+
* Thrown rather than defaulted. A record whose timestamp cannot be read has no
|
|
38
|
+
* place in a time-windowed store: filed under today it would look current
|
|
39
|
+
* forever and never prune, and filed under the epoch it would vanish from
|
|
40
|
+
* every window. `ingestDirectory` catches this and reports the file as
|
|
41
|
+
* skipped, which is the visible failure the silent ones are worth trading for.
|
|
42
|
+
*/
|
|
43
|
+
export declare function partitionOf(startedAt: string): string;
|
|
44
|
+
export interface RunBlobName {
|
|
45
|
+
readonly runId: string;
|
|
46
|
+
readonly shardKey: string;
|
|
47
|
+
/** `YYYY-MM-DD`, so it compares against an ISO timestamp with `<`. */
|
|
48
|
+
readonly date: string;
|
|
49
|
+
}
|
|
50
|
+
export declare function runBlobName(prefix: string, startedAt: string, runId: string, shardKey: string): string;
|
|
51
|
+
/** Everything a listing needs to answer, so the common queries need no downloads. */
|
|
52
|
+
export declare function parseRunBlobName(prefix: string, name: string): RunBlobName | null;
|
|
53
|
+
/** Prefix every run blob shares, so a listing can skip anything else in the container. */
|
|
54
|
+
export declare function runsPrefix(prefix: string): string;
|
|
55
|
+
//# sourceMappingURL=layout.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"layout.d.ts","sourceRoot":"","sources":["../src/layout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,yFAAyF;AACzF,eAAO,MAAM,cAAc,OAAO,CAAC;AAcnC,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAYnD;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAWnD;AAED,qBAAa,WAAY,SAAQ,KAAK;gBACxB,OAAO,EAAE,MAAM;CAI5B;AAID;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAQrD;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,WAAW,CACzB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,GACf,MAAM,CAER;AAED,qFAAqF;AACrF,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAkBjF;AAED,0FAA0F;AAC1F,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEjD"}
|