@beesolve/analytics-processor 0.3.0 → 0.5.0

package/dist/index.d.ts

@@ -7,6 +7,7 @@ declare const dimensions: readonly ["status", "country", "device", "path"];
 type Dimension = (typeof dimensions)[number];
 declare const appliedSources: readonly ["v1", "v2"];
 type AppliedSource = (typeof appliedSources)[number];
+type MarkerGrain = Grain | "sitemap";
 type TransactWriteItem = NonNullable<TransactWriteCommandInput["TransactItems"]>[number];
 type PutMembers = NonNullable<TransactWriteItem["Put"]>;
 interface TransactPutItem {
@@ -28,11 +29,11 @@ declare class AppliedMarkers {
 		readonly reverseIndexName?: string;
 	});
 	readonly isApplied: (props: {
-		readonly grain: Grain;
+		readonly grain: MarkerGrain;
 		readonly periodKey: string;
 	}) => Promise<boolean>;
 	readonly buildMarkerPut: (props: {
-		readonly grain: Grain;
+		readonly grain: MarkerGrain;
 		readonly periodKey: string;
 		readonly source: AppliedSource;
 		readonly ttl?: number;
@@ -142,6 +143,18 @@ interface HourlyAthenaResult {
 		hits: number;
 	}>;
 }
+/**
+* Start an Athena statement, poll until it succeeds, and return every result
+* row as raw string cells (header row dropped). Shared by the hourly query and
+* the sitemap monthly query so both use one polling/paging implementation.
+*/
+declare function startAndFetchRows({ athena, database, workgroup, outputLocation, queryString }: {
+	athena: Pick<AthenaClient, "send">;
+	database: string;
+	workgroup: string;
+	outputLocation?: string;
+	queryString: string;
+}): Promise<Array<Array<string | undefined>>>;
 declare function runHourlyAthenaQuery({ athena, database, table, workgroup, outputLocation, source, hourKey, topPaths }: {
 	athena: Pick<AthenaClient, "send">;
 	database: string;
@@ -212,8 +225,13 @@ declare const configSchema: v3.ObjectSchema<{
 	readonly sk: v3.LiteralSchema<"analytics", undefined>;
 	readonly startKey: v3.StringSchema<undefined>;
 	readonly startedAt: v3.StringSchema<undefined>;
+	readonly sitemapUrl: v3.OptionalSchema<v3.StringSchema<undefined>, undefined>;
 }, undefined>;
 type AnalyticsConfig = v3.InferOutput<typeof configSchema>;
+declare const sitemapConfigSchema: v3.LooseObjectSchema<{
+	readonly sitemapUrl: v3.OptionalSchema<v3.StringSchema<undefined>, undefined>;
+}, undefined>;
+type SitemapConfig = v3.InferOutput<typeof sitemapConfigSchema>;
 declare class Config {
 	private readonly props;
 	constructor(props: {
@@ -221,6 +239,10 @@ declare class Config {
 		readonly tableName: string;
 	});
 	readonly get: () => Promise<AnalyticsConfig | null>;
+	readonly getSitemapConfig: () => Promise<SitemapConfig>;
+	readonly setSitemapUrl: ({ sitemapUrl }: {
+		readonly sitemapUrl: string;
+	}) => Promise<void>;
 	/**
 	* Records the earliest ingested hour the first time any data lands, so the UI
 	* can clamp navigation to [startKey, now]. Idempotent: once `startKey` exists
@@ -362,7 +384,25 @@ interface RollupModels {
 	readonly breakdowns: Pick<Breakdowns, "buildUpdates" | "getForPeriod">;
 	readonly markers: Pick<AppliedMarkers, "isApplied" | "buildMarkerPut">;
 }
+declare function sendTransacted({ dynamo, leading, rest }: {
+	dynamo: Pick<DynamoDBDocumentClient5, "send">;
+	leading: Array<TransactItem>;
+	rest: Array<TransactItem>;
+}): Promise<ProcessHourResult>;
+declare function isMarkerRaceError(error: unknown): boolean;
 type View = Grain | "year";
+/**
+* How a bucket's value was obtained, so the UI can tell "no traffic" apart from
+* "not calculated yet":
+* - `sealed`: a finalized, rolled-up period read from its stored aggregate. A
+*   zero-hit sealed bucket is genuine "no traffic".
+* - `partial`: an in-progress period computed live from the finer grain; the
+*   value is real but may still grow.
+* - `pending`: a period past its expected processing time with no stored data
+*   and nothing computable yet — awaiting processing.
+*/
+declare const bucketStates: readonly ["sealed", "partial", "pending"];
+type BucketState = (typeof bucketStates)[number];
 interface ResolveModels {
 	readonly totals: Pick<Totals, "queryWindow" | "getOne">;
 	readonly breakdowns: Pick<Breakdowns, "getForPeriod">;
@@ -370,15 +410,21 @@ interface ResolveModels {
 }
 /**
 * Resolves one bucket's totals, computing an in-progress period on the fly from
-* the grain below it. A past period whose rollup marker is applied is read
-* directly from the stored aggregate ("sealed"); the current period, and any
-* past period still missing its marker, is summed from its children — the
-* current month folds down through the current day to the hours ingested so far.
+* exactly one grain finer, following only the current edge deeper.
+*
+* A sealed period (strictly past, rollup marker applied) is read from its stored
+* aggregate. Otherwise the period is the sum of its children at the next finer
+* grain, where every already-complete child is taken from its stored aggregate
+* and only the single in-progress child is resolved recursively:
+*
+* - partial day   = stored/queried hours `00..currentHour`
+* - partial month = stored days `01..currentDay-1` + the partial current day
+* - partial year  = stored months `01..currentMonth-1` + the partial current month
 *
-* Each level reads its children with a single ranged query (never a get per
-* child): a day sums its hours via one hourly `queryWindow`; a month sums its
-* days, and the one in-progress day among them is resolved from hours. `hour`
-* is terminal.
+* So a partial month never fans out to hours for its finished days (it trusts
+* their day aggregate) — only the one current day drills into hours. Each level
+* is one ranged query, keeping the read count bounded (a year touches ~12
+* months + ~31 days + ~24 hours) and the request sub-second.
 */
 declare function resolveTotalsLive({ models, grain, periodKey, now }: {
 	readonly models: ResolveModels;
@@ -393,6 +439,222 @@ declare function resolveBreakdownLive({ models, dimension, grain, periodKey, now
 	readonly periodKey: string;
 	readonly now: Date;
 }): Promise<Array<BreakdownRow>>;
+/**
+* Classifies a bucket as sealed / partial / pending without recomputing its
+* value. A sealed bucket is a finalized rollup (its marker is applied), so a
+* zero-hit sealed bucket is genuine "no traffic". A non-sealed bucket that
+* yielded data is in-progress (`partial`); one with nothing to show is awaiting
+* processing (`pending`).
+*/
+declare function classifyBucketState({ models, grain, periodKey, now, hasData }: {
+	readonly models: ResolveModels;
+	readonly grain: Grain;
+	readonly periodKey: string;
+	readonly now: Date;
+	readonly hasData: boolean;
+}): Promise<BucketState>;
+import { AthenaClient as AthenaClient3 } from "@aws-sdk/client-athena";
+import { DynamoDBDocumentClient as DynamoDBDocumentClient6, TransactWriteCommandInput as TransactWriteCommandInput5 } from "@aws-sdk/lib-dynamodb";
+import * as v5 from "valibot";
+type TransactWriteItem4 = NonNullable<TransactWriteCommandInput5["TransactItems"]>[number];
+type UpdateMembers3 = NonNullable<TransactWriteItem4["Update"]>;
+/**
+* Collapse a single trailing slash on a path so `/blog/` and `/blog` fold
+* together. Root `/` is preserved. Already-unslashed paths are returned as-is.
+*/
+declare function canonicalPath(path: string): string;
+/**
+* Parse an absolute sitemap `<loc>` URL and reduce it to a canonical path:
+* origin, query and fragment are stripped and a trailing slash is collapsed.
+* Throws on input the `URL` class cannot parse.
+*/
+declare function normalizeSitemapLoc(loc: string): string;
+/**
+* Extract every `<loc>` from a flat `<urlset>` sitemap, XML-unescape and
+* normalize each to a path, and dedupe preserving first-seen order (so a
+* `/blog` and `/blog/` pair collapse to one entry). Throws on a
+* `<sitemapindex>` document, which is not supported.
+*/
+declare function parseSitemap(xml: string): Array<string>;
+/**
+* Identifies this crawler's outbound sitemap requests so site operators can
+* distinguish them in their access logs.
+*/
+declare const sitemapUserAgent = "@beesolve/analytics-processor sitemap fetcher (+https://github.com/BeeSolve/packages)";
+/**
+* Fetch a sitemap over HTTP and return its normalized, deduped paths. Throws
+* with the response status if the request is not successful.
+*/
+declare function fetchSitemapPaths(url: string): Promise<Array<string>>;
+interface SitemapHourCounts {
+	readonly path: string;
+	readonly hits: number;
+}
+/**
+* Build an Athena statement that counts hits per `path` for a single UTC hour,
+* restricted to the given sitemap paths. The log-side URI is slash-collapsed in
+* SQL via {@link canonicalUriExpr} in both the `WHERE ... IN` and the
+* `GROUP BY`, mirroring {@link canonicalPath}, so `/blog` and the 301 hit on
+* `/blog/` both count against `/blog`. An empty `paths` yields a valid zero-row
+* statement (`WHERE 1 = 0`).
+*/
+declare function buildSitemapHourlyQuery({ table, source, hourKey, paths }: {
+	table: string;
+	source: AppliedSource;
+	hourKey: string;
+	paths: Array<string>;
+}): string;
+/**
+* Run {@link buildSitemapHourlyQuery} and return one {@link SitemapHourCounts}
+* per path with hits for that hour. Short-circuits to `[]` without calling
+* Athena when `paths` is empty.
+*/
+declare function runSitemapHourlyQuery({ athena, database, table, workgroup, outputLocation, source, hourKey, paths }: {
+	athena: Pick<AthenaClient3, "send">;
+	database: string;
+	table: string;
+	workgroup: string;
+	outputLocation?: string;
+	source: AppliedSource;
+	hourKey: string;
+	paths: Array<string>;
+}): Promise<Array<SitemapHourCounts>>;
+interface SitemapMonthCounts {
+	readonly path: string;
+	readonly month: string;
+	readonly days: Record<string, number>;
+	readonly hits: number;
+}
+/**
+* Build an Athena statement that counts hits per `(day-of-month, path)` across
+* a whole `YYYY-MM`, restricted to the given sitemap paths. One scan of the
+* month's partitions replaces 720 single-hour scans, which is why backfill uses
+* this instead of {@link buildSitemapHourlyQuery}. The log-side URI is
+* slash-collapsed in SQL via {@link canonicalUriExpr} in both the `WHERE ... IN`
+* and the `GROUP BY`. An empty `paths` yields a valid zero-row statement.
+*/
+declare function buildSitemapMonthlyQuery({ table, source, monthKey, paths }: {
+	table: string;
+	source: AppliedSource;
+	monthKey: string;
+	paths: Array<string>;
+}): string;
+/**
+* Run {@link buildSitemapMonthlyQuery} and reshape the rows into one
+* {@link SitemapMonthCounts} per path, with day-of-month zero-padded to `dd`.
+* Short-circuits to `[]` without calling Athena when `paths` is empty.
+*/
+declare function runSitemapMonthlyQuery({ athena, database, table, workgroup, outputLocation, source, monthKey, paths }: {
+	athena: Pick<AthenaClient3, "send">;
+	database: string;
+	table: string;
+	workgroup: string;
+	outputLocation?: string;
+	source: AppliedSource;
+	monthKey: string;
+	paths: Array<string>;
+}): Promise<Array<SitemapMonthCounts>>;
+interface ProcessSitemapModels {
+	readonly sitemapUrlStats: Pick<SitemapUrlStats, "buildAdd">;
+	readonly markers: Pick<AppliedMarkers, "isApplied" | "buildMarkerPut">;
+}
+interface ProcessSitemapAthena {
+	readonly client: Pick<AthenaClient3, "send">;
+	readonly database: string;
+	readonly table: string;
+	readonly workgroup: string;
+	readonly outputLocation?: string;
+}
+interface ProcessSitemapOptions {
+	readonly markerTtlDays?: number;
+}
+/**
+* Incrementally add a single UTC hour's per-path hits to each sitemap URL's
+* `(path, month)` item, guarded by an applied-marker so the hour is counted at
+* most once. Returns `{ status: "skipped" }` when the hour is already applied,
+* and `{ status: "applied", urls }` otherwise (`urls: 0` when nothing matched).
+*/
+declare function processSitemapHour({ hourKey, source, paths, models, athena, options, dynamo }: {
+	hourKey: string;
+	source: AppliedSource;
+	paths: Array<string>;
+	models: ProcessSitemapModels;
+	athena: ProcessSitemapAthena;
+	options?: ProcessSitemapOptions;
+	dynamo: Pick<DynamoDBDocumentClient6, "send">;
+}): Promise<{
+	status: "applied" | "skipped";
+	urls: number;
+}>;
+interface ProcessSitemapMonthModels {
+	readonly sitemapUrlStats: Pick<SitemapUrlStats, "overwriteMany">;
+}
+/**
+* Recompute a whole month's per-day hits in a single Athena scan and overwrite
+* each `(path, month)` item with `SET`. This is the backfill path: one scan of
+* the month replaces 720 single-hour scans, and `SET` is idempotent so reruns
+* are safe and no applied-marker is needed. The live hourly cron uses
+* {@link processSitemapHour} (incremental `ADD`) instead, because going forward
+* it only ever has one new hour of data. Returns `{ urls: 0 }` without calling
+* Athena when `paths` is empty.
+*/
+declare function processSitemapMonth({ monthKey, source, paths, models, athena }: {
+	monthKey: string;
+	source: AppliedSource;
+	paths: Array<string>;
+	models: ProcessSitemapMonthModels;
+	athena: ProcessSitemapAthena;
+}): Promise<{
+	urls: number;
+}>;
+declare const sitemapUrlStatSchema: v5.LooseObjectSchema<{
+	readonly pk: v5.StringSchema<undefined>;
+	readonly sk: v5.StringSchema<undefined>;
+	readonly path: v5.StringSchema<undefined>;
+	readonly month: v5.StringSchema<undefined>;
+	readonly hits: v5.NumberSchema<undefined>;
+	readonly updatedAt: v5.NumberSchema<undefined>;
+}, undefined>;
+type SitemapUrlStat = v5.InferOutput<typeof sitemapUrlStatSchema>;
+interface SitemapUrlStatRecord {
+	readonly pk: string;
+	readonly sk: string;
+	readonly path: string;
+	readonly month: string;
+	readonly hits: number;
+	readonly updatedAt: number;
+	readonly days: Record<string, number>;
+}
+declare class SitemapUrlStats {
+	private readonly props;
+	constructor(props: {
+		readonly dynamo: Pick<DynamoDBDocumentClient6, "send">;
+		readonly tableName: string;
+		readonly reverseIndexName?: string;
+	});
+	readonly buildAdd: ({ path, month, dayAttr, hits }: {
+		readonly path: string;
+		readonly month: string;
+		readonly dayAttr: string;
+		readonly hits: number;
+	}) => TransactItem;
+	readonly overwriteMany: (stats: Array<SitemapMonthCounts>) => Promise<void>;
+	readonly buildOverwrite: ({ path, month, days, hits }: {
+		readonly path: string;
+		readonly month: string;
+		readonly days: Record<string, number>;
+		readonly hits: number;
+	}) => UpdateMembers3;
+	readonly getForUrl: ({ path, startMonth, endMonth }: {
+		readonly path: string;
+		readonly startMonth: string;
+		readonly endMonth: string;
+	}) => Promise<Array<SitemapUrlStatRecord>>;
+	readonly getForMonth: ({ month }: {
+		readonly month: string;
+	}) => Promise<Array<SitemapUrlStatRecord>>;
+	private readonly parseOne;
+}
 declare function yearKey(date: Date): string;
 declare function monthKey2(date: Date): string;
 declare function dayKey2(date: Date): string;
@@ -423,4 +685,4 @@ declare function hoursBetween({ from, to }: {
 	from: Date;
 	to: Date;
 }): Array<string>;
-export { AnalyticsConfig, AppliedMarker, AppliedMarkers, AppliedSource, AthenaColumnMap, BreakdownInput, BreakdownItem, BreakdownRow, Breakdowns, Config, DeviceBucket, Dimension, Grain, HourlyAthenaResult, HourlyAthenaTotals, ProcessHourAthena, ProcessHourModels, ProcessHourOptions, ProcessHourResult, ResolveModels, SummedTotals, Totals, TotalsInput, TotalsItem, TransactItem, View, appliedMarkerSchema, appliedSources, applyDailyRollup, applyMonthlyRollup, breakdownSchema, buildHourlyQuery, chunkTransactItems, classifyDevice, columnMaps, configSchema, dayKey2 as dayKey, dimensions, grainForPeriodKey, grains, hourKey2 as hourKey, hourKeyParts, hoursBetween, maxWindowFor, monthKey2 as monthKey, parsePeriodKey, processHour, requiredRecordFields, resolveBreakdownLive, resolveTotalsLive, runHourlyAthenaQuery, shiftAnchor, sumBreakdown, sumTotals, topNWithOther, totalsSchema, windowKeys, yearKey };
+export { AnalyticsConfig, AppliedMarker, AppliedMarkers, AppliedSource, AthenaColumnMap, BreakdownInput, BreakdownItem, BreakdownRow, Breakdowns, BucketState, Config, DeviceBucket, Dimension, Grain, HourlyAthenaResult, HourlyAthenaTotals, MarkerGrain, ProcessHourAthena, ProcessHourModels, ProcessHourOptions, ProcessHourResult, ProcessSitemapAthena, ProcessSitemapModels, ProcessSitemapMonthModels, ProcessSitemapOptions, ResolveModels, SitemapConfig, SitemapHourCounts, SitemapMonthCounts, SitemapUrlStat, SitemapUrlStatRecord, SitemapUrlStats, SummedTotals, Totals, TotalsInput, TotalsItem, TransactItem, View, appliedMarkerSchema, appliedSources, applyDailyRollup, applyMonthlyRollup, breakdownSchema, bucketStates, buildHourlyQuery, buildSitemapHourlyQuery, buildSitemapMonthlyQuery, canonicalPath, chunkTransactItems, classifyBucketState, classifyDevice, columnMaps, configSchema, dayKey2 as dayKey, dimensions, fetchSitemapPaths, grainForPeriodKey, grains, hourKey2 as hourKey, hourKeyParts, hoursBetween, isMarkerRaceError, maxWindowFor, monthKey2 as monthKey, normalizeSitemapLoc, parsePeriodKey, parseSitemap, processHour, processSitemapHour, processSitemapMonth, requiredRecordFields, resolveBreakdownLive, resolveTotalsLive, runHourlyAthenaQuery, runSitemapHourlyQuery, runSitemapMonthlyQuery, sendTransacted, shiftAnchor, sitemapUrlStatSchema, sitemapUserAgent, startAndFetchRows, sumBreakdown, sumTotals, topNWithOther, totalsSchema, windowKeys, yearKey };