@cloudbitmaps/tools 0.0.0-stage → 0.20.0

package/dist/cost.d.ts

@@ -0,0 +1,353 @@
+/** Pluggable rate card. Rates differ by cloud/region/term and drift over time; the formulas don't. */
+export interface PricingProfile {
+    readonly name: string;
+    readonly storage: {
+        /** Object GET (per **million** requests). A ranged GET bills as a full GET. */
+        readonly getPerMillion: number;
+        /** Object PUT (per million) — what a load pays, per request. */
+        readonly putPerMillion: number;
+        readonly storagePerGiBMonth: number;
+        /**
+         * Requests one pointer read costs: a read of a segment's registry row, whose version comes back with its bytes.
+         * Default **1**, and 1 on S3, GCS and Azure Blob, each of which answers it with one GET. Charged for each operand
+         * of an intersection, for the pointer reads a load makes, and for each pointer refresh. Set it when a
+         * registry of your own takes more than one request to read a row.
+         */
+        readonly requestsPerPointerRead?: number;
+        /**
+         * Requests one tail read costs: the read of a segment's index from the end of its generation, which needs the
+         * object's size. Default **1**, S3's, whose suffix-range GET returns the size with the bytes, and GCS's, the same.
+         * **2** on Azure Blob, which takes no suffix range and reads the properties and then the bytes. Charged for each
+         * operand of an intersection. A load reads no index of its own: a row that carries a summary of its current
+         * generation gives the load the size it needs, and the checks it makes are one metadata request on every backend.
+         * A chunk read knows its range, and is one request everywhere.
+         */
+        readonly requestsPerSizedRead?: number;
+    };
+    /**
+     * The always-on Redis the verdict compares against: exactly one of the two. `sizedToData`, the default's, prices
+     * the cheapest cluster that holds the report's stored bytes; `monthlyUSD` is one cluster you name, whatever the
+     * data size. A profile carrying both is refused rather than read one way.
+     */
+    readonly redis: {
+        readonly monthlyUSD: number;
+        readonly sizedToData?: never;
+    } | {
+        readonly sizedToData: RedisSizing;
+        readonly monthlyUSD?: never;
+    };
+}
+/** One node type a Redis cluster can be built from, as its cloud prices it. */
+export interface RedisNodeType {
+    readonly name: string;
+    /** Memory, in GiB, as the cloud publishes it for the node type. */
+    readonly memoryGiB: number;
+    /**
+     * SSD, in GiB, on a data-tiering node (ElastiCache's `r6gd`): keys stay in memory, and the values read least
+     * recently move to the SSD. Counted in full beside the memory, as AWS counts a data-tiering node's capacity.
+     */
+    readonly ssdGiB?: number;
+    /** Price per node-hour, in USD. */
+    readonly hourlyUSD: number;
+    /**
+     * The most shards a cluster of this type is priced at. The default catalogue prices its burstable nodes as one
+     * shard: a cluster sharded across them is not how data anyone queries is held. Default: no limit.
+     */
+    readonly maxShards?: number;
+}
+/**
+ * Redis sized to hold the data: enough shards for the bytes, each a primary and its replicas, on whichever node
+ * type in `nodeTypes` makes that cheapest. Prices within a millionth of a dollar of the cheapest are a tie, which
+ * fewer nodes win, then the lower price, then the name, so the answer never depends on the order of the rows. The data is held at its stored,
+ * compressed size, which is a floor on the memory Redis needs for it: a native Redis bitmap is sized by its highest
+ * id, not by how many ids it holds, so sparse ids need more.
+ */
+export interface RedisSizing {
+    /** Where the prices come from, and when; the report's notes quote it. */
+    readonly source: string;
+    readonly nodeTypes: readonly RedisNodeType[];
+    /** Replicas per shard. AWS's best practice is 2; Multi-AZ needs at least 1. */
+    readonly replicasPerShard: number;
+    /** The share of each node's memory kept back from data: ElastiCache's `reserved-memory-percent`, 25% by default. */
+    readonly reservedMemoryFraction: number;
+}
+/**
+ * **ElastiCache for Redis OSS in us-east-1, on-demand**: node prices from AWS's public price list, version
+ * 20260914063714 (published 2026-09-14, effective 2026-09-01,
+ * https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonElastiCache/20260914063714/us-east-1/index.json), and the
+ * memory AWS publishes for each node type. Every shard is a primary and two replicas, AWS's best practice, with
+ * ElastiCache's default 25% of each node's memory reserved, and the burstable `t4g` nodes are priced as one shard.
+ *
+ * It lists only rows that are the cheapest fit for some size of data: the data-tiering `r6gd` rows above about
+ * 30 GiB, and the larger in-memory `r6g` rows, `2xlarge` and up, when those are left out. The current `m7g` and `r7g` nodes are not here,
+ * because an `m6g` or `r6g` with the same memory costs less. It is the cheapest cluster of THIS kind; Redis bought
+ * another way costs less — one replica a shard a third less, ElastiCache for Valkey 20% less a node, reserved nodes
+ * less again — so against those the saving the verdict reports is smaller. Pass their prices to compare with them.
+ */
+export declare const ELASTICACHE_REDIS_US_EAST_1_ONDEMAND: RedisSizing;
+/**
+ * One ElastiCache HA cluster, a primary and two replicas of cache.m7g.large — the m7g family's smallest node —
+ * at $0.158 an hour each: 3 × 730 h × $0.158 = $346.02, published to the dollar. The per-request crossovers on the
+ * benchmarks page are drawn against this one cluster, whatever the data size; pass it as `pricing.redis` to compare
+ * with it. It is not the cheapest cluster for any size of data: three cache.m6g.large hold the same memory for less.
+ */
+export declare const ONE_REDIS_HA_CLUSTER: Readonly<{
+    monthlyUSD: number;
+}>;
+/**
+ * Default profile — **AWS us-east-1, on-demand**, mid-2026, from the fact-checked published pricing rather than
+ * copied from a blog post, with Redis sized to the data. Override it for your region, cloud, or committed term.
+ * Its prices are as old as the release of this package that ships them: a planning figure, not a quote.
+ */
+export declare const AWS_US_EAST_1_ONDEMAND: PricingProfile;
+/** Sustained access pattern. All rates default to 0; unspecified ⇒ that op contributes nothing. */
+export interface Workload {
+    /**
+     * Point reads (`has`) per second. Each cache miss is one object GET, once the segment's pointer and index are
+     * read.
+     */
+    readonly readsPerSec?: number;
+    readonly intersectsPerSec?: number;
+    /** Cache hit rate in `[0, 1]` — hits are free; only misses cost. Default 0. */
+    readonly cacheHitRate?: number;
+    /**
+     * Range requests one intersection makes for its chunks, summed over its operands. Default 1. The engine reads each
+     * operand's needed chunks (the chunk-skipping survivors) as coalesced ranges: chunks within 256 KiB of each other
+     * are read in one request, up to 1 MiB, so this is the requests the chunks need, not the chunks. It follows the
+     * bytes the needed chunks span in each object: two segments sharing 100 chunks that sit side by side make one request
+     * of each, `2`; the same 100 spread over each object make as many as it takes to cover most of the object, a
+     * request per MiB, and read most of its bytes. Count it by running the engine over a layout of your own, as
+     * `bench/range-counts.cjs` does, or take it as one request per MiB the shared chunks span, per operand. The model adds each operand's pointer and index reads itself (see
+     * {@link Workload.operandsPerIntersect}), so count chunk requests only. One more GET per operand whose index
+     * outgrows the reader's tail read (256 KiB by default), which then reads the index whole, belongs here too, and so
+     * does a pointer re-read by an intersect slow enough to outlive {@link Workload.genTtlMs}.
+     */
+    readonly chunksPerIntersect?: number;
+    /**
+     * Segments each intersection reads, `exclude` operands included; at least 1. Default 2. An intersection is priced
+     * **cold**: before its chunks, each operand's pointer is read, then its index, in one read of the object's tail —
+     * 2 GETs an operand on S3 and GCS, so a cold intersect that makes `r` range requests of each of two segments is `4 + 2r` GETs there,
+     * and 3 on Azure Blob, whose tail read is two requests (see {@link PricingProfile}). `cacheHitRate`
+     * does not apply to intersections, so a long-lived reader that answers a repeat from its cache pays less. Other
+     * combines read their operands the same way and can be priced here too, with the chunk requests they make.
+     */
+    readonly operandsPerIntersect?: number;
+    /**
+     * Generations published per month across the modeled data — the write side of a loaded store. Default **0**
+     * ⇒ loads are not modeled and the report *discloses* the omission rather than silently under-reporting.
+     */
+    readonly loadsPerMonth?: number;
+    /**
+     * PUT-class requests one load's object write issues, each priced at the PUT rate. Default **1** (a single-object
+     * PUT). A multipart write of `P` parts bills `P + 2` (initiate, the parts, complete) — set it when you know your
+     * object sizes. The model adds what `store.load()` does around the write, at any `keep` from 1 to 64: the pointer's
+     * write, PUT-class on S3, and four GETs: the pointer read twice, one check that the next generation number is
+     * free, and one that the current generation's object is there, which tells the collection that deletes by name it
+     * may. The current generation's size comes from the row's summary of it, so the load reads no index. Collection
+     * deletes the generation the window pushed out by name, so it lists only on every 16th generation, which adds a
+     * PUT-class request and two pointer reads there and makes no check that the current generation is there, a
+     * sixteenth of each on average. That is a segment with two
+     * generations behind it, whose row carries a summary; its first two loads make fewer requests and collect nothing,
+     * and the first load onto a row with no summary of its current generation (one a rollback left when it could not
+     * open the key, one a registry of your own stores without one, or one this store cannot use) reads that generation's index, one tail read, in
+     * place of the check that its object is there. A publish that loses a race to another writer
+     * reads the pointer again, and a load whose check finds the number taken (a crashed load's object, or the
+     * generations a rollback left above the pointer) lists the segment's objects to number past them and to collect,
+     * two PUT-class requests on S3 and two more pointer reads. So does every load whose `keep` is above 64, which lists
+     * to collect: one more PUT-class request and two more pointer reads than the model counts. The counts
+     * are a cleartext segment's: an encrypted segment's load reads its row once more, after its ids and before it
+     * unwraps the key, one more GET ($0.40 per million at the default prices) that the model leaves out, beside the
+     * key-management calls it does not price either.
+     */
+    readonly requestsPerLoad?: number;
+    /**
+     * Segments each long-lived reader process keeps reading. A reader re-reads a segment's pointer when it reads the
+     * segment after {@link Workload.genTtlMs} has passed, so each costs at most one GET per `genTtlMs` in each process
+     * that reads it — 1,314,000 GETs a month at the default 2 s — and the whole term at most one GET per point read
+     * (`readsPerSec`, across the fleet). Intersections are priced cold and pay their own pointer reads. Default **0**
+     * ⇒ the refresh is not modeled, and the report *discloses* the omission when there are reads to refresh for.
+     *
+     * It assumes each hot segment stays open in the reader's cache (`cache.readerMax`, 1,024 segments by default,
+     * and `cache.readerMaxBytes`, 64 MiB of parsed index). A read of a segment the cache evicted that needs its index or
+     * a chunk the chunk cache does not hold opens it again, a tail read, which is not priced here; with a timed refresh the store keeps the segment's resolution apart from its
+     * reader, so the eviction reads its pointer again only for a segment whose resolution was let go too, and with none
+     * every reopen reads it. Neither is the re-open every reader makes after each load priced, for the new generation's
+     * index. Size those caches to keep the hot set open; the report says so when `hotSegments` is more than a store
+     * keeps open by default.
+     */
+    readonly hotSegments?: number;
+    /**
+     * Long-lived reader processes, each keeping its own {@link Workload.hotSegments} open and refreshing their
+     * pointers on its own: ten processes reading the same hundred segments pay ten times one process's refresh.
+     * At least 1. Default 1.
+     */
+    readonly readerProcesses?: number;
+    /**
+     * How long the reader trusts a pointer, in ms: the store's `cache.genTtlMs`. Default 2000, the store's own
+     * default; pass the store's when it sets one. `0` turns the timed refresh off, and the model bills none: a
+     * reader then re-reads a pointer only when its cache evicts the segment, a read finds its generation swept, or a
+     * write invalidates it, none of which this term prices.
+     */
+    readonly genTtlMs?: number;
+    /**
+     * Segments the retention sweep (`retireExpired`) retires a month. Each costs 7 registry reads, 2 writes and a
+     * delete with {@link Workload.conditionalDelete} on, and 6 reads and 2 writes with it off, priced at the GET and PUT
+     * rates, a delete unbilled, as S3 leaves it. Default
+     * **0**.
+     */
+    readonly retirementsPerMonth?: number;
+    /**
+     * Tombstones the sweep purges a month: at a steady state, as many as it retires a month, a `tombstoneGraceMs` later.
+     * A purge costs 4 reads and 2 deletes with {@link Workload.conditionalDelete} on, and 3 reads and a write with it
+     * off. Default **0**.
+     */
+    readonly purgesPerMonth?: number;
+    /**
+     * Whether the registry's deletes remove a row for good (`RegCaps.conditionalDelete`), which the shipped registries
+     * do where the backend applies a delete precondition. Default **true**. `false` prices the sweep of a registry that
+     * only tombstones: a purge rewrites the row instead of deleting it, and every later full sweep reads each tombstone
+     * left (two reads per purged segment), which this term does not price.
+     */
+    readonly conditionalDelete?: boolean;
+}
+/** One (group of) segment(s) for planning. `count` = how many like this (default 1). */
+export interface SegmentSizing {
+    readonly sizeBytes?: number;
+    readonly cardinality?: number;
+    /** Number of segments with these characteristics. Default 1. */
+    readonly count?: number;
+}
+export interface EstimateInput {
+    readonly segments: readonly SegmentSizing[];
+    readonly workload?: Workload;
+    readonly pricing?: PricingProfile;
+}
+export interface CostReport {
+    readonly monthlyUSD: {
+        readonly byOp: {
+            readonly reads: number;
+            readonly intersects: number;
+            readonly storage: number;
+            /**
+             * Loads: each its object's `requestsPerLoad` PUT-class requests plus what `store.load()` adds around them.
+             * 0 unless `loadsPerMonth` is set.
+             */
+            readonly loads: number;
+            /** The pointer refresh of {@link Workload.hotSegments}. 0 unless `hotSegments` is set. */
+            readonly pointerRefresh: number;
+            /**
+             * The retention sweep's retirements and purges: each's registry reads at the GET rate and its writes at the PUT
+             * rate. 0 unless `retirementsPerMonth` or `purgesPerMonth` is set.
+             */
+            readonly retention: number;
+        };
+        readonly total: number;
+    };
+    /**
+     * The Redis the verdict compares against. Sized, the default, it is the cheapest cluster in
+     * `pricing.redis.sizedToData` that holds this report's stored bytes; fixed, it is `pricing.redis.monthlyUSD` as
+     * given.
+     *
+     * **It is not additive across reports.** Each report sizes Redis to its own bytes, so a per-segment report
+     * (`groundedReport` of one segment's size) compares with a cluster holding that one segment alone, and the baselines of a store's
+     * segments do not sum to the store's. To judge a store, price all its segments in one `estimateCost`. To alarm on
+     * one, sum `monthlyUSD.total` over its segments and compare the sum with the Redis you would run for it: a
+     * per-segment verdict against that whole price fires only when one segment alone costs more than all of it.
+     */
+    readonly redisBaseline: {
+        readonly basis: 'fixed';
+        readonly monthlyUSD: number;
+    } | {
+        readonly basis: 'sized-to-data';
+        readonly monthlyUSD: number;
+        /** The node type, the shards, the nodes (every shard's primary and replicas), and whether they tier to SSD. */
+        readonly cluster: {
+            readonly nodeType: string;
+            readonly shards: number;
+            readonly nodes: number;
+            readonly dataTiering: boolean;
+        };
+    };
+    /**
+     * Sustained read rate at which the pay-per-use model's cost passes {@link CostReport.redisBaseline}, **evaluated
+     * at this report's `cacheHitRate`** — so a higher cache-hit rate raises it (cache hits are free) — with the other
+     * request axes at 0, and with what keeping the data readable costs taken out of the baseline first: storage, and
+     * this report's pointer refresh. Loads, the write side, are left out of it. `Infinity` means it never crosses (a
+     * 100% cache-hit rate). The published anchor (~329 reads/s) is against {@link ONE_REDIS_HA_CLUSTER}, at
+     * `cacheHitRate: 0` with no refresh modeled.
+     */
+    readonly redisCrossover: {
+        readonly readsPerSec: number;
+    };
+    readonly verdict: 'win-big' | 'win' | 'lose-zone';
+    readonly rationale: string;
+    readonly assumptions: {
+        readonly cacheHitRate: number;
+        readonly pricingName: string;
+        /**
+         * True when the size was measured (a `groundedReport` given a byte count), false for a pure `estimateCost` and
+         * for a `groundedReport` given `storageBytes: null`.
+         */
+        readonly grounded: boolean;
+        /** What the model assumed and what it left out, in words. The last says how the Redis was priced. */
+        readonly notes: readonly string[];
+    };
+}
+/**
+ * What the retention sweep makes per segment, counted with a store that counts its requests
+ * (`tests/core/retention-hard-purge.test.ts` holds each figure to the engine, and `tests/tools/cost.test.ts` holds the
+ * estimator to these). With the registry's `conditionalDelete` on, a retirement files a due-index pointer to the
+ * tombstone and a purge removes the row and every pointer: a later sweep reads nothing of a purged segment. With it
+ * off, a retirement files no pointer, a purge rewrites the row as a tombstone, and every later full sweep reads what
+ * is left, two objects per purged segment.
+ */
+export declare const RETENTION_SWEEP_REQUESTS: Readonly<{
+    conditionalDelete: {
+        retirement: {
+            reads: number;
+            writes: number;
+            deletes: number;
+        };
+        purge: {
+            reads: number;
+            writes: number;
+            deletes: number;
+        };
+    };
+    tombstoning: {
+        retirement: {
+            reads: number;
+            writes: number;
+            deletes: number;
+        };
+        purge: {
+            reads: number;
+            writes: number;
+            deletes: number;
+        };
+    };
+}>;
+/**
+ * **Planning** cost estimate — pure, no instance or live data needed (sizing, sales, what-if). Segment sizes
+ * are taken as given (or roughly derived from cardinality); use {@link groundedReport} with a segment's measured
+ * size for exact, real sizes. See {@link CostReport}.
+ */
+export declare function estimateCost(input: EstimateInput): CostReport;
+/**
+ * **Grounded** report from a measured byte total + a supplied workload. A segment handle's `stat()` reports the
+ * size of its current generation as `size`, read from the `.crbm` footer and index with no payload reads:
+ *
+ * ```ts
+ * const report = groundedReport({ storageBytes: (await seg.stat()).sizeBytes, workload: { readsPerSec: 50 } });
+ * ```
+ *
+ * Sum the sizes of several segments to price them together; see {@link CostReport.redisBaseline} for why one report
+ * of the sum is not the sum of the reports. `storageBytes: null`, which `stat()` answers for a segment with no
+ * generation and on a store whose source cannot report a size, prices storage at $0 and says so in the notes, with
+ * `assumptions.grounded` false: nothing was measured, so the report does not claim a confident $0.
+ */
+export declare function groundedReport(input: {
+    readonly storageBytes: number | null;
+    readonly workload?: Workload;
+    readonly pricing?: PricingProfile;
+}): CostReport;

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * `@cloudbitmaps/tools` — offline tools for CloudBitmaps that need nothing internal from a store.
+ *
+ * The cost model: {@link estimateCost} prices a store you plan, from the segment sizes and the workload you give it,
+ * and {@link groundedReport} one you run, from the bytes it stores, which a segment handle's `stat()` reports as
+ * `size`. Each report compares the store's monthly cost on object storage with an always-on Redis, and says where
+ * Redis wins. It is a planning tool: the price lists here are as old as the release that ships them, so pass your
+ * own for your region, your cloud or a committed term.
+ *
+ * Nothing here reads or writes a store. It takes `ValidationError` from `@cloudbitmaps/core`'s public entry, so a
+ * caller classifies the model's refusals as it does the library's.
+ */
+export { estimateCost, groundedReport, AWS_US_EAST_1_ONDEMAND, ELASTICACHE_REDIS_US_EAST_1_ONDEMAND, ONE_REDIS_HA_CLUSTER, } from './cost.js';
+export type { PricingProfile, RedisNodeType, RedisSizing, CostReport, Workload, SegmentSizing, EstimateInput, } from './cost.js';