@polar-sh/sdk 1.0.2 → 1.0.3-alpha.3

package/dist/outpost/index.cjs

@@ -0,0 +1,587 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_rolldown_runtime = require("../rolldown-runtime-C0BPl7ul.cjs");
+const require_base = require("../base-DHy1aJ70.cjs");
+let ioredis = require("ioredis");
+let node_crypto = require("node:crypto");
+let node_events = require("node:events");
+let ws = require("ws");
+ws = require_rolldown_runtime.__toESM(ws, 1);
+//#region src/outpost/event.ts
+const customerKey = (input) => {
+	if (input.customer_id === void 0 === (input.external_customer_id === void 0)) throw new TypeError("Provide exactly one customer_id or external_customer_id");
+	const identifier = input.customer_id ?? input.external_customer_id;
+	return `${input.customer_id !== void 0 ? "customer" : "external"}:${identifier}`;
+};
+const parseEvent = (input) => {
+	if (input.timestamp !== void 0 && !/(Z|[+-]\d{2}:?\d{2})$/i.test(input.timestamp)) throw new TypeError("Event timestamps must include a timezone");
+	const timestamp = input.timestamp === void 0 ? Date.now() : Date.parse(input.timestamp);
+	const fraction = input.timestamp?.match(/\.(\d+)/)?.[1] ?? "";
+	const microseconds = timestamp * 1e3 + Number(fraction.padEnd(6, "0").slice(3, 6));
+	if (!Number.isSafeInteger(microseconds)) throw new TypeError("Invalid event timestamp");
+	return {
+		name: input.name,
+		customer: customerKey(input),
+		timestamp: microseconds,
+		metadata: input.metadata ?? {}
+	};
+};
+const getProperty = (event, property) => {
+	if (property === "name") return event.name;
+	if (property === "source") return "user";
+	if (property === "timestamp") return Math.trunc(event.timestamp / 1e6);
+	let value = event.metadata;
+	for (const part of property.replace(/^metadata\./, "").split(".")) {
+		if (value === null || typeof value !== "object" || Array.isArray(value) || !Object.hasOwn(value, part)) return void 0;
+		value = value[part];
+	}
+	return value;
+};
+//#endregion
+//#region src/outpost/reducer.ts
+const emptyStatistics = () => ({
+	count: 0,
+	sum: 0,
+	min: null,
+	max: null
+});
+const addStatistics = (target, other) => {
+	const total = target.sum + other.sum;
+	if (!Number.isFinite(total)) throw new TypeError("Aggregation values must be finite");
+	target.count += other.count;
+	target.sum = total;
+	if (other.min !== null) target.min = target.min === null ? other.min : Math.min(target.min, other.min);
+	if (other.max !== null) target.max = target.max === null ? other.max : Math.max(target.max, other.max);
+};
+const quantity = (statistics, func) => {
+	switch (func) {
+		case "count": return statistics.count;
+		case "sum": return statistics.sum;
+		case "min": return statistics.min ?? 0;
+		case "max": return statistics.max ?? 0;
+		case "avg": return statistics.count ? statistics.sum / statistics.count : 0;
+	}
+};
+const BUCKET_SIZE = 3e8;
+const getBucketStart = (timestamp) => {
+	if (!Number.isSafeInteger(timestamp)) throw new TypeError("Invalid bucket timestamp");
+	return Math.floor(timestamp / BUCKET_SIZE) * BUCKET_SIZE;
+};
+const getMatcher = (filter) => {
+	if (!("property" in filter)) {
+		const children = filter.clauses.map(getMatcher);
+		return filter.conjunction === "and" ? (event) => children.every((match) => match(event)) : (event) => children.some((match) => match(event));
+	}
+	return (event) => {
+		const actual = getProperty(event, filter.property);
+		if (actual === void 0 || actual === null) return false;
+		if (filter.operator === "eq" || filter.operator === "ne") {
+			const actualValue = typeof actual === "boolean" ? Number(actual) : actual;
+			const expectedValue = typeof filter.value === "boolean" ? Number(filter.value) : filter.value;
+			return filter.operator === "eq" ? actualValue === expectedValue : actualValue !== expectedValue;
+		}
+		if (typeof actual !== typeof filter.value) return false;
+		if (typeof actual !== "number" && typeof actual !== "string" || typeof filter.value !== "number" && typeof filter.value !== "string") return false;
+		switch (filter.operator) {
+			case "gt": return actual > filter.value;
+			case "gte": return actual >= filter.value;
+			case "lt": return actual < filter.value;
+			case "lte": return actual <= filter.value;
+		}
+	};
+};
+const reduce = (reducers, events, oldestBucketStart) => {
+	const updates = /* @__PURE__ */ new Map();
+	const matchers = reducers.map((reducer) => [reducer, getMatcher(reducer.filter)]);
+	for (const event of events) {
+		const bucketStart = getBucketStart(event.timestamp);
+		if (bucketStart < oldestBucketStart) continue;
+		for (const [reducer, match] of matchers) {
+			if (!match(event)) continue;
+			const { aggregation } = reducer;
+			const value = aggregation.func === "count" ? 1 : getProperty(event, aggregation.property);
+			if (typeof value !== "number") continue;
+			if (!Number.isFinite(value)) throw new TypeError("Aggregation values must be finite");
+			const key = JSON.stringify([
+				event.customer,
+				reducer.id,
+				bucketStart
+			]);
+			let update = updates.get(key);
+			if (!update) {
+				update = {
+					customer: event.customer,
+					reducer: reducer.id,
+					bucketStart,
+					statistics: emptyStatistics()
+				};
+				updates.set(key, update);
+			}
+			addStatistics(update.statistics, aggregation.func === "count" ? {
+				count: 1,
+				sum: 0,
+				min: null,
+				max: null
+			} : {
+				count: 1,
+				sum: value,
+				min: value,
+				max: value
+			});
+		}
+	}
+	return [...updates.values()];
+};
+//#endregion
+//#region src/outpost/scripts.ts
+const READ_STATE_SCRIPT = `
+if not redis.call('HGET', KEYS[1], 'epoch') then
+    redis.call('HSET', KEYS[1], 'epoch', ARGV[1], 'revision', '0')
+end
+return redis.call('HGETALL', KEYS[1])
+`;
+const WRITE_BUCKET_SCRIPT = `
+local cutoff = redis.call('HGET', KEYS[1], 'cold_until')
+if cutoff and tonumber(ARGV[4]) < tonumber(cutoff) then
+    return redis.call('HGETALL', KEYS[1])
+end
+local prefix = 'bucket:' .. ARGV[4] .. ':'
+local total = tonumber(redis.call('HGET', KEYS[1], prefix .. 'sum') or '0') + tonumber(ARGV[6])
+assert(total == total and math.abs(total) ~= math.huge, 'Aggregation values must be finite')
+redis.call('HINCRBYFLOAT', KEYS[1], prefix .. 'sum', ARGV[6])
+redis.call('HINCRBY', KEYS[1], prefix .. 'count', ARGV[5])
+for i, func in ipairs({'min', 'max'}) do
+    local raw = ARGV[6 + i]
+    if raw ~= '' then
+        local value = tonumber(raw)
+        assert(value and value == value and math.abs(value) ~= math.huge, 'Invalid value')
+        local stored = redis.call('HGET', KEYS[1], prefix .. func)
+        local previous = stored and tonumber(stored)
+        if not previous or (func == 'min' and value < previous)
+            or (func == 'max' and value > previous) then
+            redis.call('HSET', KEYS[1], prefix .. func, raw)
+        end
+    end
+end
+
+if not redis.call('HGET', KEYS[1], 'epoch') then
+    redis.call('HSET', KEYS[1], 'epoch', ARGV[3], 'revision', '0')
+end
+redis.call('HINCRBY', KEYS[1], 'revision', 1)
+local state = redis.call('HGETALL', KEYS[1])
+redis.call('PUBLISH', ARGV[2], cjson.encode({customer = ARGV[1], state = state}))
+return state
+`;
+const APPLY_SNAPSHOT_SCRIPT = `
+local cutoff = tonumber(ARGV[4])
+local current = redis.call('HGET', KEYS[1], 'cold_until')
+if current and tonumber(current) > cutoff then
+    return redis.call('HGETALL', KEYS[1])
+end
+local version = tonumber(ARGV[5])
+local previous = redis.call('HGET', KEYS[1], 'snapshot_version')
+if not previous or version >= tonumber(previous) then
+    for _, field in ipairs(redis.call('HKEYS', KEYS[1])) do
+        if string.sub(field, 1, 5) == 'cold:'
+            or (string.sub(field, 1, 7) == 'bucket:'
+                and tonumber(string.match(field, '^bucket:(%-?%d+):')) < cutoff) then
+            redis.call('HDEL', KEYS[1], field)
+        end
+    end
+    redis.call('HSET', KEYS[1], 'cold_until', ARGV[4], 'snapshot_version', ARGV[5],
+        'cold:count', ARGV[8], 'cold:sum', ARGV[9])
+    if ARGV[10] ~= '' then redis.call('HSET', KEYS[1], 'cold:min', ARGV[10]) end
+    if ARGV[11] ~= '' then redis.call('HSET', KEYS[1], 'cold:max', ARGV[11]) end
+end
+local credit_version = redis.call('HGET', KEYS[1], 'credited_version:' .. ARGV[6])
+if not credit_version or version >= tonumber(credit_version) then
+    redis.call('HSET', KEYS[1], 'credited:' .. ARGV[6], ARGV[7],
+        'credited_version:' .. ARGV[6], ARGV[5])
+end
+
+if not redis.call('HGET', KEYS[1], 'epoch') then
+    redis.call('HSET', KEYS[1], 'epoch', ARGV[3], 'revision', '0')
+end
+redis.call('HINCRBY', KEYS[1], 'revision', 1)
+local state = redis.call('HGETALL', KEYS[1])
+redis.call('PUBLISH', ARGV[2], cjson.encode({customer = ARGV[1], state = state}))
+return state
+`;
+//#endregion
+//#region src/outpost/client.ts
+const CACHE_SIZE = 1024;
+const SYNC_INTERVAL = 3e4;
+const TIMEOUT = 5e3;
+const generateEpoch = () => (0, node_crypto.randomUUID)().replaceAll("-", "");
+const number = (value) => {
+	const parsed = JSON.parse(value);
+	if (!Number.isFinite(parsed)) throw new TypeError("Stored aggregation values must be finite numbers");
+	return parsed;
+};
+const signal = () => {
+	let resolve;
+	return {
+		promise: new Promise((done) => {
+			resolve = done;
+		}),
+		resolve
+	};
+};
+const withTimeout = async (promise) => {
+	let timer;
+	try {
+		return await Promise.race([promise, new Promise((_, reject) => {
+			timer = setTimeout(() => reject(new require_base.PolarError("Outpost operation timed out")), TIMEOUT);
+		})]);
+	} finally {
+		clearTimeout(timer);
+	}
+};
+const locked = (local, action) => {
+	const result = local.pending.then(action);
+	local.pending = result.catch(() => {});
+	return result;
+};
+const decodeState = (value) => {
+	const fields = {};
+	for (let i = 0; i < value.length; i += 2) fields[value[i]] = value[i + 1];
+	const state = {
+		epoch: fields.epoch,
+		revision: BigInt(fields.revision),
+		coldUntil: fields.cold_until === void 0 ? void 0 : number(fields.cold_until),
+		cold: emptyStatistics(),
+		buckets: /* @__PURE__ */ new Map(),
+		credited: {}
+	};
+	for (const [field, value] of Object.entries(fields)) if (field.startsWith("cold:")) {
+		const statistic = field.slice(5);
+		state.cold[statistic] = number(value);
+	} else if (field.startsWith("bucket:")) {
+		const [, timestamp, statistic] = field.split(":");
+		const start = Number(timestamp);
+		const bucket = state.buckets.get(start) ?? emptyStatistics();
+		bucket[statistic] = number(value);
+		state.buckets.set(start, bucket);
+	} else if (field.startsWith("credited:")) state.credited[field.slice(9)] = number(value);
+	return state;
+};
+const customerArguments = (customer) => {
+	const separator = customer.indexOf(":");
+	return { [customer.startsWith("customer:") ? "customer_id" : "external_customer_id"]: customer.slice(separator + 1) };
+};
+var Outpost = class {
+	redisUrl;
+	options;
+	redis;
+	subscriber;
+	websocket;
+	reducers = [];
+	cache = /* @__PURE__ */ new Map();
+	actors = /* @__PURE__ */ new Set();
+	configured = signal();
+	synced = signal();
+	isSynced = false;
+	generation = 0;
+	channel = "";
+	refreshTimer;
+	failure;
+	messages = Promise.resolve();
+	synchronizing;
+	constructor(redisUrl, options) {
+		this.redisUrl = redisUrl;
+		this.options = options;
+	}
+	async start() {
+		if (this.redis) throw new require_base.PolarError("Outpost is already connected");
+		this.failure = void 0;
+		this.configured = signal();
+		this.synced = signal();
+		this.messages = Promise.resolve();
+		try {
+			this.redis = new ioredis.Redis(this.redisUrl, {
+				lazyConnect: true,
+				enableOfflineQueue: false,
+				autoResendUnfulfilledCommands: false,
+				maxRetriesPerRequest: 0,
+				commandTimeout: TIMEOUT
+			});
+			this.redis.defineCommand("outpostReadState", {
+				numberOfKeys: 1,
+				lua: READ_STATE_SCRIPT
+			});
+			this.redis.defineCommand("outpostWriteBucket", {
+				numberOfKeys: 1,
+				lua: WRITE_BUCKET_SCRIPT
+			});
+			this.redis.defineCommand("outpostApplySnapshot", {
+				numberOfKeys: 1,
+				lua: APPLY_SNAPSHOT_SCRIPT
+			});
+			const redis = this.redis;
+			this.redis.on("error", () => {});
+			this.redis.on("close", () => {
+				if (this.redis === redis) this.invalidate();
+			});
+			this.redis.on("ready", () => {
+				if (this.subscriber?.status === "ready") this.synchronize();
+			});
+			await withTimeout(this.redis.connect());
+			await this.redis.ping();
+			this.channel = `outpost:${this.redis.options.db}:updates`;
+			this.subscriber = this.redis.duplicate({
+				lazyConnect: true,
+				autoResubscribe: false
+			});
+			const subscriber = this.subscriber;
+			this.subscriber.on("error", () => {});
+			this.subscriber.on("close", () => {
+				if (this.subscriber === subscriber) this.invalidate();
+			});
+			this.subscriber.on("ready", () => {
+				this.synchronize();
+			});
+			this.subscriber.on("message", (_channel, data) => {
+				this.receiveRedis(data).catch((error) => this.fail(error));
+			});
+			await withTimeout(this.subscriber.connect());
+			await withTimeout(this.synced.promise);
+			const [url, request] = this.options.client.buildRequest("GET", "/v1/outpost/");
+			const endpoint = new URL(url);
+			endpoint.protocol = endpoint.protocol === "https:" ? "wss:" : "ws:";
+			this.websocket = new ws.default(endpoint, {
+				headers: Object.fromEntries(new Headers(request.headers)),
+				handshakeTimeout: TIMEOUT
+			});
+			this.websocket.on("error", (error) => this.fail(error));
+			this.websocket.on("close", () => {
+				if (this.websocket) this.fail(new require_base.PolarError("Outpost WebSocket disconnected"));
+			});
+			this.websocket.on("message", (data) => {
+				this.messages = this.messages.then(() => this.receiveWebsocket(data.toString())).catch((error) => this.fail(error));
+			});
+			await withTimeout((0, node_events.once)(this.websocket, "open"));
+			await this.send({ type: "configuration" });
+			this.refreshTimer = setInterval(() => {
+				if (this.synchronizing) return;
+				this.synchronize();
+			}, SYNC_INTERVAL);
+			this.requireRedis();
+			return this;
+		} catch (error) {
+			await this.close();
+			throw error;
+		}
+	}
+	async close() {
+		clearInterval(this.refreshTimer);
+		const websocket = this.websocket;
+		const connections = [this.subscriber, this.redis].filter((redis) => redis !== void 0);
+		this.websocket = void 0;
+		this.subscriber = void 0;
+		this.redis = void 0;
+		const closed = connections.map((redis) => {
+			const ended = redis.status === "end" ? Promise.resolve() : (0, node_events.once)(redis, "end");
+			redis.disconnect();
+			return ended;
+		});
+		if (websocket && websocket.readyState !== ws.default.CLOSED) {
+			closed.push((0, node_events.once)(websocket, "close"));
+			websocket.terminate();
+		}
+		this.invalidate();
+		this.synced.resolve();
+		this.configured.resolve();
+		await Promise.allSettled([
+			...closed,
+			this.messages,
+			this.synchronizing
+		]);
+		this.cache.clear();
+		this.actors.clear();
+		this.reducers = [];
+	}
+	async [Symbol.asyncDispose]() {
+		await this.close();
+	}
+	async ingest({ events }) {
+		const redis = this.requireRedis();
+		if (!events.length) return {
+			inserted: 0,
+			duplicates: 0
+		};
+		const parsed = events.map(parseEvent);
+		await this.waitConfigured();
+		const updates = reduce(this.reducers, parsed, parsed.reduce((oldest, event) => Math.min(oldest, getBucketStart(event.timestamp)), Infinity));
+		for (const customer of new Set(parsed.map((event) => event.customer))) await this.actor(customerArguments(customer));
+		for (const update of updates) {
+			const identity = `${update.customer}:${update.reducer}`;
+			const statistics = update.statistics;
+			const state = await redis.outpostWriteBucket(`outpost:state:${identity}`, identity, this.channel, generateEpoch(), update.bucketStart, statistics.count, statistics.sum, statistics.min ?? "", statistics.max ?? "");
+			await this.receiveState(identity, decodeState(state));
+		}
+		return {
+			inserted: parsed.length,
+			duplicates: 0
+		};
+	}
+	async customerMeter(input) {
+		const customer = customerKey(input);
+		this.requireRedis();
+		await this.waitConfigured();
+		const reducer = this.reducers.find((item) => item.meter_ids?.includes(input.meter_id));
+		if (!reducer) return null;
+		const state = (await this.warmCustomer(customer, reducer)).state;
+		if (!state || state.coldUntil === void 0) throw new require_base.PolarError("Customer snapshot is not ready");
+		const combined = { ...state.cold };
+		for (const [start, statistics] of state.buckets) if (start >= state.coldUntil) addStatistics(combined, statistics);
+		const consumed = quantity(combined, reducer.aggregation.func);
+		const credited = state.credited[input.meter_id] ?? 0;
+		return {
+			...input,
+			consumed_units: consumed,
+			credited_units: credited,
+			balance: credited - consumed
+		};
+	}
+	async actor(input) {
+		await this.waitConfigured();
+		for (const reducer of this.reducers) await this.warmCustomer(customerKey(input), reducer);
+	}
+	requireRedis() {
+		if (this.failure) throw this.failure;
+		if (!this.redis) throw new require_base.PolarError("Call Outpost.start() before using Outpost");
+		return this.redis;
+	}
+	async waitConfigured() {
+		await withTimeout(this.configured.promise);
+		this.requireRedis();
+	}
+	async send(message) {
+		this.requireRedis();
+		if (!this.websocket || this.websocket.readyState !== ws.default.OPEN) throw new require_base.PolarError("Outpost WebSocket is not connected");
+		await new Promise((resolve, reject) => this.websocket.send(JSON.stringify(message), (error) => error ? reject(error) : resolve()));
+	}
+	async warmCustomer(customer, reducer) {
+		const identity = `${customer}:${reducer.id}`;
+		this.requireRedis();
+		while (!this.isSynced) {
+			await withTimeout(this.synced.promise);
+			this.requireRedis();
+		}
+		let local = this.cache.get(identity);
+		if (!local) {
+			local = { pending: Promise.resolve() };
+			this.cache.set(identity, local);
+			if (this.cache.size > CACHE_SIZE) {
+				const evicted = this.cache.keys().next().value;
+				this.cache.delete(evicted);
+				this.actors.delete(evicted);
+			}
+		}
+		this.cache.delete(identity);
+		this.cache.set(identity, local);
+		if (local.state && this.actors.has(identity)) return local;
+		const entry = local;
+		await locked(entry, async () => {
+			if (!entry.state) entry.state = await this.loadState(identity);
+			if (!this.actors.has(identity)) {
+				for (const meter_id of reducer.meter_ids ?? []) await this.send({
+					type: "meter_usage",
+					payload: {
+						...customerArguments(customer),
+						meter_id
+					}
+				});
+				this.actors.add(identity);
+			}
+		});
+		return entry;
+	}
+	async loadState(customer) {
+		const epoch = generateEpoch();
+		return decodeState(await this.requireRedis().outpostReadState(`outpost:state:${customer}`, epoch));
+	}
+	async receiveState(customer, state) {
+		const local = this.cache.get(customer);
+		if (!local) return;
+		await locked(local, async () => {
+			const current = local.state;
+			if (!current) return;
+			if (current.epoch !== state.epoch) state = await this.loadState(customer);
+			else if (current.revision >= state.revision) return;
+			local.state = state;
+		});
+	}
+	async receiveRedis(data) {
+		const update = JSON.parse(data);
+		await this.receiveState(update.customer, decodeState(update.state));
+	}
+	synchronize() {
+		if (this.synchronizing) return this.synchronizing;
+		const generation = this.generation;
+		this.synchronizing = (async () => {
+			if (this.failure || !this.subscriber || this.subscriber.status !== "ready" || this.redis?.status !== "ready") return;
+			if (!this.isSynced) await this.subscriber.subscribe(this.channel);
+			const entries = Array.from(this.cache);
+			for (const [customer, local] of entries) await locked(local, async () => {
+				const state = await this.loadState(customer);
+				if (generation === this.generation) local.state = state;
+			});
+			if (generation === this.generation) {
+				this.isSynced = true;
+				this.synced.resolve();
+			}
+		})().catch((error) => {
+			if (error instanceof TypeError) this.fail(error);
+			else this.invalidate();
+		}).finally(() => {
+			this.synchronizing = void 0;
+			if (!this.failure && generation !== this.generation && this.redis?.status === "ready" && this.subscriber?.status === "ready") this.synchronize();
+		});
+		return this.synchronizing;
+	}
+	invalidate() {
+		this.generation++;
+		this.isSynced = false;
+		this.synced.resolve();
+		this.synced = signal();
+		for (const local of this.cache.values()) local.state = void 0;
+	}
+	fail(error) {
+		this.failure = error instanceof Error ? error : new require_base.PolarError(String(error));
+		this.configured.resolve();
+		this.synced.resolve();
+	}
+	async receiveWebsocket(data) {
+		const message = JSON.parse(data);
+		if (message.type === "configuration") {
+			this.reducers = message.payload.reducers;
+			this.configured.resolve();
+		} else if (message.type === "meter_usage") {
+			const snapshot = message.payload;
+			const reducer = this.reducers.find((item) => item.meter_ids?.includes(snapshot.meter_id));
+			if (!reducer) return;
+			if (snapshot.customer_id !== null && snapshot.external_customer_id !== null) return;
+			const identity = `${customerKey({
+				customer_id: snapshot.customer_id ?? void 0,
+				external_customer_id: snapshot.external_customer_id ?? void 0
+			})}:${reducer.id}`;
+			if (!Number.isSafeInteger(snapshot.version) || !Number.isSafeInteger(snapshot.cold_until)) throw new TypeError("Snapshot timestamps must be safe integer microseconds");
+			for (const value of [
+				snapshot.count,
+				snapshot.sum,
+				snapshot.min,
+				snapshot.max,
+				snapshot.credited_units
+			]) if (value !== null && !Number.isFinite(value)) throw new TypeError("Snapshot values must be finite");
+			const state = await this.requireRedis().outpostApplySnapshot(`outpost:state:${identity}`, identity, this.channel, generateEpoch(), snapshot.cold_until, snapshot.version, snapshot.meter_id, snapshot.credited_units, snapshot.count, snapshot.sum, snapshot.min ?? "", snapshot.max ?? "");
+			await this.receiveState(identity, decodeState(state));
+		}
+	}
+};
+//#endregion
+exports.Outpost = Outpost;
+exports.getBucketStart = getBucketStart;
+exports.getMatcher = getMatcher;
+exports.reduce = reduce;
+
+//# sourceMappingURL=index.cjs.map