@beesolve/analytics-processor 0.0.0-stage → 0.2.0

package/dist/index.d.ts

@@ -0,0 +1,359 @@
+export * from "@beesolve/ip-info";
+import { DynamoDBDocumentClient, TransactWriteCommandInput } from "@aws-sdk/lib-dynamodb";
+import * as v from "valibot";
+declare const grains: readonly ["hour", "day", "month"];
+type Grain = (typeof grains)[number];
+declare const dimensions: readonly ["status", "country", "device", "path"];
+type Dimension = (typeof dimensions)[number];
+declare const appliedSources: readonly ["v1", "v2"];
+type AppliedSource = (typeof appliedSources)[number];
+type TransactWriteItem = NonNullable<TransactWriteCommandInput["TransactItems"]>[number];
+type PutMembers = NonNullable<TransactWriteItem["Put"]>;
+interface TransactPutItem {
+	readonly Put: PutMembers;
+}
+declare const appliedMarkerSchema: v.ObjectSchema<{
+	readonly pk: v.StringSchema<undefined>;
+	readonly sk: v.StringSchema<undefined>;
+	readonly source: v.PicklistSchema<readonly ["v1", "v2"], undefined>;
+	readonly appliedAt: v.NumberSchema<undefined>;
+	readonly ttl: v.OptionalSchema<v.NumberSchema<undefined>, undefined>;
+}, undefined>;
+type AppliedMarker = v.InferOutput<typeof appliedMarkerSchema>;
+declare class AppliedMarkers {
+	private readonly props;
+	constructor(props: {
+		readonly dynamo: Pick<DynamoDBDocumentClient, "send">;
+		readonly tableName: string;
+		readonly reverseIndexName?: string;
+	});
+	readonly isApplied: (props: {
+		readonly grain: Grain;
+		readonly periodKey: string;
+	}) => Promise<boolean>;
+	readonly buildMarkerPut: (props: {
+		readonly grain: Grain;
+		readonly periodKey: string;
+		readonly source: AppliedSource;
+		readonly ttl?: number;
+	}) => TransactPutItem;
+}
+import { AthenaClient } from "@aws-sdk/client-athena";
+import { DynamoDBDocumentClient as DynamoDBDocumentClient2, TransactWriteCommandInput as TransactWriteCommandInput2 } from "@aws-sdk/lib-dynamodb";
+import * as v2 from "valibot";
+type TransactWriteItem2 = NonNullable<TransactWriteCommandInput2["TransactItems"]>[number];
+type UpdateMembers = NonNullable<TransactWriteItem2["Update"]>;
+interface TransactUpdateItem {
+	readonly Update: UpdateMembers;
+}
+interface TotalsInput {
+	readonly timeKey: string;
+	readonly hits: number;
+	readonly success: number;
+	readonly errors: number;
+	readonly errors4xx: number;
+	readonly errors5xx: number;
+	readonly bytesSent: number;
+	readonly bytesReceived: number;
+	readonly p50TimeTaken?: number;
+	readonly p95TimeTaken?: number;
+	readonly cacheHitRatio?: number;
+	readonly uniqueIpCount?: number;
+}
+declare const totalsSchema: v2.ObjectSchema<{
+	readonly pk: v2.StringSchema<undefined>;
+	readonly sk: v2.StringSchema<undefined>;
+	readonly grain: v2.PicklistSchema<readonly ["hour", "day", "month"], undefined>;
+	readonly timeKey: v2.StringSchema<undefined>;
+	readonly hits: v2.NumberSchema<undefined>;
+	readonly success: v2.NumberSchema<undefined>;
+	readonly errors: v2.NumberSchema<undefined>;
+	readonly errors4xx: v2.NumberSchema<undefined>;
+	readonly errors5xx: v2.NumberSchema<undefined>;
+	readonly bytesSent: v2.NumberSchema<undefined>;
+	readonly bytesReceived: v2.NumberSchema<undefined>;
+	readonly p50TimeTaken: v2.OptionalSchema<v2.NumberSchema<undefined>, undefined>;
+	readonly p95TimeTaken: v2.OptionalSchema<v2.NumberSchema<undefined>, undefined>;
+	readonly cacheHitRatio: v2.OptionalSchema<v2.NumberSchema<undefined>, undefined>;
+	readonly uniqueIpCount: v2.OptionalSchema<v2.NumberSchema<undefined>, undefined>;
+	readonly ttl: v2.OptionalSchema<v2.NumberSchema<undefined>, undefined>;
+}, undefined>;
+type TotalsItem = v2.InferOutput<typeof totalsSchema>;
+declare class Totals {
+	private readonly props;
+	constructor(props: {
+		readonly dynamo: Pick<DynamoDBDocumentClient2, "send">;
+		readonly tableName: string;
+		readonly reverseIndexName?: string;
+	});
+	readonly addHourly: (props: {
+		readonly item: TotalsInput;
+		readonly ttl: number;
+	}) => Promise<void>;
+	readonly addAggregate: (props: {
+		readonly grain: "day" | "month";
+		readonly item: TotalsInput;
+	}) => Promise<void>;
+	readonly buildHourlyUpdate: (props: {
+		readonly item: TotalsInput;
+		readonly ttl: number;
+	}) => TransactUpdateItem;
+	readonly buildAggregateUpdate: (props: {
+		readonly grain: "day" | "month";
+		readonly item: TotalsInput;
+	}) => TransactUpdateItem;
+	readonly queryWindow: (props: {
+		readonly grain: Grain;
+		readonly startKey: string;
+		readonly endKey: string;
+	}) => Promise<Array<TotalsItem>>;
+	readonly getOne: (props: {
+		readonly grain: Grain;
+		readonly timeKey: string;
+	}) => Promise<TotalsItem | undefined>;
+	private readonly buildUpdateMembers;
+	private readonly parseOne;
+}
+interface BreakdownRow {
+	readonly value: string;
+	readonly hits: number;
+	readonly bytes: number;
+}
+interface SummedTotals {
+	readonly hits: number;
+	readonly success: number;
+	readonly errors: number;
+	readonly errors4xx: number;
+	readonly errors5xx: number;
+	readonly bytesSent: number;
+	readonly bytesReceived: number;
+}
+declare function sumTotals(items: Array<TotalsInput>): SummedTotals;
+declare function sumBreakdown(items: Array<BreakdownRow>): Array<BreakdownRow>;
+declare function topNWithOther({ items, n, otherValue }: {
+	items: Array<BreakdownRow>;
+	n: number;
+	otherValue: string;
+}): Array<BreakdownRow>;
+/**
+* Normalized Glue/Parquet column names the processor's hourly query reads. Pass
+* these as `CloudFrontAccessLoggingSettings`' `athena.recordFields` to deliver
+* only the columns the processor needs, shrinking Parquet and cutting Athena cost.
+*/
+declare const requiredRecordFields: readonly ["timestamp", "sc_status", "sc_bytes", "cs_bytes", "time_taken", "x_edge_result_type", "x_edge_detailed_result_type", "cs_uri_stem", "c_ip", "cs_user_agent"];
+interface AthenaColumnMap {
+	readonly status: string;
+	readonly bytesSent: string;
+	readonly bytesReceived: string;
+	readonly timeTaken: string;
+	readonly resultType: string;
+	readonly detailedResultType: string;
+	readonly uriStem: string;
+	readonly clientIp: string;
+	readonly userAgent: string;
+	readonly timestamp: string;
+}
+/**
+* Per-source SQL column references — the single source of truth for how each
+* logical field is addressed in each table.
+*
+* v2 is the Parquet Glue table: Athena normalizes Parquet column names to
+* lowercase with hyphens folded to underscores, and numeric columns are already
+* typed, so no casting is needed.
+*
+* v1 is the legacy tab-delimited LazySimpleSerDe table: its column names contain
+* hyphens and parentheses, so they must be double-quoted, and every numeric
+* column is typed as string in the Glue schema, so arithmetic references must
+* cast explicitly.
+*/
+declare const columnMaps: Record<AppliedSource, AthenaColumnMap>;
+interface HourParts {
+	readonly year: string;
+	readonly month: string;
+	readonly day: string;
+	readonly hour: string;
+}
+declare function hourKeyParts(hourKey: string): HourParts;
+/**
+* Builds ONE Athena statement that scans a single UTC hour and returns, in a
+* single pass, the grand-total aggregate row plus per-status, per-device,
+* per-path, and per-ip grouped rows via GROUPING SETS.
+*
+* Why no per-set LIMIT: GROUPING SETS cannot apply an independent LIMIT to each
+* set within one statement. Rather than fragment the scan, the full grouped
+* rows are returned and {@link runHourlyAthenaQuery} applies the `topPaths`
+* top-N (folding the remainder into `__other__`) downstream via
+* {@link topNWithOther}. `topPaths` is still threaded through this builder so
+* callers have a single source of truth for the value.
+*/
+declare function buildHourlyQuery({ table, source, hourKey, topPaths: _topPaths }: {
+	table: string;
+	source: AppliedSource;
+	hourKey: string;
+	topPaths: number;
+}): string;
+interface HourlyAthenaTotals {
+	readonly hits: number;
+	readonly success: number;
+	readonly errors: number;
+	readonly errors4xx: number;
+	readonly errors5xx: number;
+	readonly bytesSent: number;
+	readonly bytesReceived: number;
+	readonly p50TimeTaken: number;
+	readonly p95TimeTaken: number;
+	readonly cacheHitRatio: number;
+	readonly uniqueIpCount: number;
+}
+interface HourlyAthenaResult {
+	readonly totals: HourlyAthenaTotals;
+	readonly byStatus: Array<BreakdownRow>;
+	readonly byDevice: Array<BreakdownRow>;
+	readonly byPath: Array<BreakdownRow>;
+	readonly topIps: Array<{
+		ip: string;
+		hits: number;
+	}>;
+}
+declare function runHourlyAthenaQuery({ athena, database, table, workgroup, outputLocation, source, hourKey, topPaths }: {
+	athena: Pick<AthenaClient, "send">;
+	database: string;
+	table: string;
+	workgroup: string;
+	outputLocation?: string;
+	source: AppliedSource;
+	hourKey: string;
+	topPaths: number;
+}): Promise<HourlyAthenaResult>;
+import { DynamoDBDocumentClient as DynamoDBDocumentClient3, TransactWriteCommandInput as TransactWriteCommandInput3 } from "@aws-sdk/lib-dynamodb";
+import * as v3 from "valibot";
+type TransactWriteItem3 = NonNullable<TransactWriteCommandInput3["TransactItems"]>[number];
+type UpdateMembers2 = NonNullable<TransactWriteItem3["Update"]>;
+interface TransactUpdateItem2 {
+	readonly Update: UpdateMembers2;
+}
+interface BreakdownInput {
+	readonly value: string;
+	readonly hits: number;
+	readonly bytes: number;
+}
+declare const breakdownSchema: v3.ObjectSchema<{
+	readonly pk: v3.StringSchema<undefined>;
+	readonly sk: v3.StringSchema<undefined>;
+	readonly dimension: v3.PicklistSchema<readonly ["status", "country", "device", "path"], undefined>;
+	readonly grain: v3.PicklistSchema<readonly ["hour", "day", "month"], undefined>;
+	readonly periodKey: v3.StringSchema<undefined>;
+	readonly value: v3.StringSchema<undefined>;
+	readonly hits: v3.NumberSchema<undefined>;
+	readonly bytes: v3.NumberSchema<undefined>;
+	readonly ttl: v3.OptionalSchema<v3.NumberSchema<undefined>, undefined>;
+}, undefined>;
+type BreakdownItem = v3.InferOutput<typeof breakdownSchema>;
+declare class Breakdowns {
+	private readonly props;
+	constructor(props: {
+		readonly dynamo: Pick<DynamoDBDocumentClient3, "send">;
+		readonly tableName: string;
+		readonly reverseIndexName?: string;
+	});
+	readonly addMany: (props: {
+		readonly dimension: Dimension;
+		readonly grain: Grain;
+		readonly periodKey: string;
+		readonly items: Array<BreakdownInput>;
+		readonly ttl?: number;
+	}) => Promise<void>;
+	readonly buildUpdates: (props: {
+		readonly dimension: Dimension;
+		readonly grain: Grain;
+		readonly periodKey: string;
+		readonly items: Array<BreakdownInput>;
+		readonly ttl?: number;
+	}) => Array<TransactUpdateItem2>;
+	readonly getForPeriod: (props: {
+		readonly dimension: Dimension;
+		readonly grain: Grain;
+		readonly periodKey: string;
+	}) => Promise<Array<BreakdownItem>>;
+	private readonly buildUpdateMembers;
+	private readonly parseOne;
+}
+type DeviceBucket = "mobile" | "desktop" | "bot" | "other";
+declare function classifyDevice(userAgent: string): DeviceBucket;
+import { AthenaClient as AthenaClient2 } from "@aws-sdk/client-athena";
+import { DynamoDBDocumentClient as DynamoDBDocumentClient4, TransactWriteCommandInput as TransactWriteCommandInput4 } from "@aws-sdk/lib-dynamodb";
+import { IpInfoCache } from "@beesolve/ip-info";
+type TransactItem = NonNullable<TransactWriteCommandInput4["TransactItems"]>[number];
+interface ProcessHourModels {
+	readonly totals: Pick<Totals, "buildHourlyUpdate">;
+	readonly breakdowns: Pick<Breakdowns, "buildUpdates">;
+	readonly markers: Pick<AppliedMarkers, "isApplied" | "buildMarkerPut">;
+	readonly ipCache: Pick<IpInfoCache, "enrichMany">;
+}
+interface ProcessHourAthena {
+	readonly client: Pick<AthenaClient2, "send">;
+	readonly database: string;
+	readonly table: string;
+	readonly workgroup: string;
+	readonly outputLocation?: string;
+}
+interface ProcessHourOptions {
+	readonly topPaths: number;
+	readonly hourlyTtlDays: number;
+	readonly ipInfoApiKey?: string;
+}
+interface ProcessHourResult {
+	readonly status: "applied" | "skipped";
+}
+declare function processHour({ hourKey, source, models, athena, options, dynamo }: {
+	hourKey: string;
+	source: AppliedSource;
+	models: ProcessHourModels;
+	athena: ProcessHourAthena;
+	options: ProcessHourOptions;
+	dynamo: Pick<DynamoDBDocumentClient4, "send">;
+}): Promise<ProcessHourResult>;
+declare function applyDailyRollup({ dayKey, models, dynamo }: {
+	dayKey: string;
+	models: RollupModels;
+	dynamo: Pick<DynamoDBDocumentClient4, "send">;
+}): Promise<ProcessHourResult>;
+declare function applyMonthlyRollup({ monthKey, models, dynamo }: {
+	monthKey: string;
+	models: RollupModels;
+	dynamo: Pick<DynamoDBDocumentClient4, "send">;
+}): Promise<ProcessHourResult>;
+declare function chunkTransactItems({ leading, rest, limit }: {
+	leading: Array<TransactItem>;
+	rest: Array<TransactItem>;
+	limit?: number;
+}): Array<Array<TransactItem>>;
+interface RollupModels {
+	readonly totals: Pick<Totals, "buildAggregateUpdate" | "queryWindow">;
+	readonly breakdowns: Pick<Breakdowns, "buildUpdates" | "getForPeriod">;
+	readonly markers: Pick<AppliedMarkers, "isApplied" | "buildMarkerPut">;
+}
+declare function yearKey(date: Date): string;
+declare function monthKey2(date: Date): string;
+declare function dayKey2(date: Date): string;
+declare function hourKey2(date: Date): string;
+declare function maxWindowFor(grain: Grain): number;
+declare function shiftAnchor({ grain, anchor, size, direction }: {
+	grain: Grain;
+	anchor: Date;
+	size: number;
+	direction: "next" | "prev";
+}): Date;
+declare function windowKeys({ grain, anchor, size }: {
+	grain: Grain;
+	anchor: Date;
+	size: number;
+}): {
+	startKey: string;
+	endKey: string;
+};
+declare function hoursBetween({ from, to }: {
+	from: Date;
+	to: Date;
+}): Array<string>;
+export { AppliedMarker, AppliedMarkers, AppliedSource, AthenaColumnMap, BreakdownInput, BreakdownItem, BreakdownRow, Breakdowns, DeviceBucket, Dimension, Grain, HourlyAthenaResult, HourlyAthenaTotals, ProcessHourAthena, ProcessHourModels, ProcessHourOptions, ProcessHourResult, SummedTotals, Totals, TotalsInput, TotalsItem, TransactItem, appliedMarkerSchema, appliedSources, applyDailyRollup, applyMonthlyRollup, breakdownSchema, buildHourlyQuery, chunkTransactItems, classifyDevice, columnMaps, dayKey2 as dayKey, dimensions, grains, hourKey2 as hourKey, hourKeyParts, hoursBetween, maxWindowFor, monthKey2 as monthKey, processHour, requiredRecordFields, runHourlyAthenaQuery, shiftAnchor, sumBreakdown, sumTotals, topNWithOther, totalsSchema, windowKeys, yearKey };