@beesolve/analytics-processor 0.4.0 → 0.5.0

package/dist/index.d.ts

@@ -7,6 +7,7 @@ declare const dimensions: readonly ["status", "country", "device", "path"];
 type Dimension = (typeof dimensions)[number];
 declare const appliedSources: readonly ["v1", "v2"];
 type AppliedSource = (typeof appliedSources)[number];
+type MarkerGrain = Grain | "sitemap";
 type TransactWriteItem = NonNullable<TransactWriteCommandInput["TransactItems"]>[number];
 type PutMembers = NonNullable<TransactWriteItem["Put"]>;
 interface TransactPutItem {
@@ -28,11 +29,11 @@ declare class AppliedMarkers {
 		readonly reverseIndexName?: string;
 	});
 	readonly isApplied: (props: {
-		readonly grain: Grain;
+		readonly grain: MarkerGrain;
 		readonly periodKey: string;
 	}) => Promise<boolean>;
 	readonly buildMarkerPut: (props: {
-		readonly grain: Grain;
+		readonly grain: MarkerGrain;
 		readonly periodKey: string;
 		readonly source: AppliedSource;
 		readonly ttl?: number;
@@ -142,6 +143,18 @@ interface HourlyAthenaResult {
 		hits: number;
 	}>;
 }
+/**
+* Start an Athena statement, poll until it succeeds, and return every result
+* row as raw string cells (header row dropped). Shared by the hourly query and
+* the sitemap monthly query so both use one polling/paging implementation.
+*/
+declare function startAndFetchRows({ athena, database, workgroup, outputLocation, queryString }: {
+	athena: Pick<AthenaClient, "send">;
+	database: string;
+	workgroup: string;
+	outputLocation?: string;
+	queryString: string;
+}): Promise<Array<Array<string | undefined>>>;
 declare function runHourlyAthenaQuery({ athena, database, table, workgroup, outputLocation, source, hourKey, topPaths }: {
 	athena: Pick<AthenaClient, "send">;
 	database: string;
@@ -212,8 +225,13 @@ declare const configSchema: v3.ObjectSchema<{
 	readonly sk: v3.LiteralSchema<"analytics", undefined>;
 	readonly startKey: v3.StringSchema<undefined>;
 	readonly startedAt: v3.StringSchema<undefined>;
+	readonly sitemapUrl: v3.OptionalSchema<v3.StringSchema<undefined>, undefined>;
 }, undefined>;
 type AnalyticsConfig = v3.InferOutput<typeof configSchema>;
+declare const sitemapConfigSchema: v3.LooseObjectSchema<{
+	readonly sitemapUrl: v3.OptionalSchema<v3.StringSchema<undefined>, undefined>;
+}, undefined>;
+type SitemapConfig = v3.InferOutput<typeof sitemapConfigSchema>;
 declare class Config {
 	private readonly props;
 	constructor(props: {
@@ -221,6 +239,10 @@ declare class Config {
 		readonly tableName: string;
 	});
 	readonly get: () => Promise<AnalyticsConfig | null>;
+	readonly getSitemapConfig: () => Promise<SitemapConfig>;
+	readonly setSitemapUrl: ({ sitemapUrl }: {
+		readonly sitemapUrl: string;
+	}) => Promise<void>;
 	/**
 	* Records the earliest ingested hour the first time any data lands, so the UI
 	* can clamp navigation to [startKey, now]. Idempotent: once `startKey` exists
@@ -362,6 +384,12 @@ interface RollupModels {
 	readonly breakdowns: Pick<Breakdowns, "buildUpdates" | "getForPeriod">;
 	readonly markers: Pick<AppliedMarkers, "isApplied" | "buildMarkerPut">;
 }
+declare function sendTransacted({ dynamo, leading, rest }: {
+	dynamo: Pick<DynamoDBDocumentClient5, "send">;
+	leading: Array<TransactItem>;
+	rest: Array<TransactItem>;
+}): Promise<ProcessHourResult>;
+declare function isMarkerRaceError(error: unknown): boolean;
 type View = Grain | "year";
 /**
 * How a bucket's value was obtained, so the UI can tell "no traffic" apart from
@@ -425,6 +453,208 @@ declare function classifyBucketState({ models, grain, periodKey, now, hasData }:
 	readonly now: Date;
 	readonly hasData: boolean;
 }): Promise<BucketState>;
+import { AthenaClient as AthenaClient3 } from "@aws-sdk/client-athena";
+import { DynamoDBDocumentClient as DynamoDBDocumentClient6, TransactWriteCommandInput as TransactWriteCommandInput5 } from "@aws-sdk/lib-dynamodb";
+import * as v5 from "valibot";
+type TransactWriteItem4 = NonNullable<TransactWriteCommandInput5["TransactItems"]>[number];
+type UpdateMembers3 = NonNullable<TransactWriteItem4["Update"]>;
+/**
+* Collapse a single trailing slash on a path so `/blog/` and `/blog` fold
+* together. Root `/` is preserved. Already-unslashed paths are returned as-is.
+*/
+declare function canonicalPath(path: string): string;
+/**
+* Parse an absolute sitemap `<loc>` URL and reduce it to a canonical path:
+* origin, query and fragment are stripped and a trailing slash is collapsed.
+* Throws on input the `URL` class cannot parse.
+*/
+declare function normalizeSitemapLoc(loc: string): string;
+/**
+* Extract every `<loc>` from a flat `<urlset>` sitemap, XML-unescape and
+* normalize each to a path, and dedupe preserving first-seen order (so a
+* `/blog` and `/blog/` pair collapse to one entry). Throws on a
+* `<sitemapindex>` document, which is not supported.
+*/
+declare function parseSitemap(xml: string): Array<string>;
+/**
+* Identifies this crawler's outbound sitemap requests so site operators can
+* distinguish them in their access logs.
+*/
+declare const sitemapUserAgent = "@beesolve/analytics-processor sitemap fetcher (+https://github.com/BeeSolve/packages)";
+/**
+* Fetch a sitemap over HTTP and return its normalized, deduped paths. Throws
+* with the response status if the request is not successful.
+*/
+declare function fetchSitemapPaths(url: string): Promise<Array<string>>;
+interface SitemapHourCounts {
+	readonly path: string;
+	readonly hits: number;
+}
+/**
+* Build an Athena statement that counts hits per `path` for a single UTC hour,
+* restricted to the given sitemap paths. The log-side URI is slash-collapsed in
+* SQL via {@link canonicalUriExpr} in both the `WHERE ... IN` and the
+* `GROUP BY`, mirroring {@link canonicalPath}, so `/blog` and the 301 hit on
+* `/blog/` both count against `/blog`. An empty `paths` yields a valid zero-row
+* statement (`WHERE 1 = 0`).
+*/
+declare function buildSitemapHourlyQuery({ table, source, hourKey, paths }: {
+	table: string;
+	source: AppliedSource;
+	hourKey: string;
+	paths: Array<string>;
+}): string;
+/**
+* Run {@link buildSitemapHourlyQuery} and return one {@link SitemapHourCounts}
+* per path with hits for that hour. Short-circuits to `[]` without calling
+* Athena when `paths` is empty.
+*/
+declare function runSitemapHourlyQuery({ athena, database, table, workgroup, outputLocation, source, hourKey, paths }: {
+	athena: Pick<AthenaClient3, "send">;
+	database: string;
+	table: string;
+	workgroup: string;
+	outputLocation?: string;
+	source: AppliedSource;
+	hourKey: string;
+	paths: Array<string>;
+}): Promise<Array<SitemapHourCounts>>;
+interface SitemapMonthCounts {
+	readonly path: string;
+	readonly month: string;
+	readonly days: Record<string, number>;
+	readonly hits: number;
+}
+/**
+* Build an Athena statement that counts hits per `(day-of-month, path)` across
+* a whole `YYYY-MM`, restricted to the given sitemap paths. One scan of the
+* month's partitions replaces 720 single-hour scans, which is why backfill uses
+* this instead of {@link buildSitemapHourlyQuery}. The log-side URI is
+* slash-collapsed in SQL via {@link canonicalUriExpr} in both the `WHERE ... IN`
+* and the `GROUP BY`. An empty `paths` yields a valid zero-row statement.
+*/
+declare function buildSitemapMonthlyQuery({ table, source, monthKey, paths }: {
+	table: string;
+	source: AppliedSource;
+	monthKey: string;
+	paths: Array<string>;
+}): string;
+/**
+* Run {@link buildSitemapMonthlyQuery} and reshape the rows into one
+* {@link SitemapMonthCounts} per path, with day-of-month zero-padded to `dd`.
+* Short-circuits to `[]` without calling Athena when `paths` is empty.
+*/
+declare function runSitemapMonthlyQuery({ athena, database, table, workgroup, outputLocation, source, monthKey, paths }: {
+	athena: Pick<AthenaClient3, "send">;
+	database: string;
+	table: string;
+	workgroup: string;
+	outputLocation?: string;
+	source: AppliedSource;
+	monthKey: string;
+	paths: Array<string>;
+}): Promise<Array<SitemapMonthCounts>>;
+interface ProcessSitemapModels {
+	readonly sitemapUrlStats: Pick<SitemapUrlStats, "buildAdd">;
+	readonly markers: Pick<AppliedMarkers, "isApplied" | "buildMarkerPut">;
+}
+interface ProcessSitemapAthena {
+	readonly client: Pick<AthenaClient3, "send">;
+	readonly database: string;
+	readonly table: string;
+	readonly workgroup: string;
+	readonly outputLocation?: string;
+}
+interface ProcessSitemapOptions {
+	readonly markerTtlDays?: number;
+}
+/**
+* Incrementally add a single UTC hour's per-path hits to each sitemap URL's
+* `(path, month)` item, guarded by an applied-marker so the hour is counted at
+* most once. Returns `{ status: "skipped" }` when the hour is already applied,
+* and `{ status: "applied", urls }` otherwise (`urls: 0` when nothing matched).
+*/
+declare function processSitemapHour({ hourKey, source, paths, models, athena, options, dynamo }: {
+	hourKey: string;
+	source: AppliedSource;
+	paths: Array<string>;
+	models: ProcessSitemapModels;
+	athena: ProcessSitemapAthena;
+	options?: ProcessSitemapOptions;
+	dynamo: Pick<DynamoDBDocumentClient6, "send">;
+}): Promise<{
+	status: "applied" | "skipped";
+	urls: number;
+}>;
+interface ProcessSitemapMonthModels {
+	readonly sitemapUrlStats: Pick<SitemapUrlStats, "overwriteMany">;
+}
+/**
+* Recompute a whole month's per-day hits in a single Athena scan and overwrite
+* each `(path, month)` item with `SET`. This is the backfill path: one scan of
+* the month replaces 720 single-hour scans, and `SET` is idempotent so reruns
+* are safe and no applied-marker is needed. The live hourly cron uses
+* {@link processSitemapHour} (incremental `ADD`) instead, because going forward
+* it only ever has one new hour of data. Returns `{ urls: 0 }` without calling
+* Athena when `paths` is empty.
+*/
+declare function processSitemapMonth({ monthKey, source, paths, models, athena }: {
+	monthKey: string;
+	source: AppliedSource;
+	paths: Array<string>;
+	models: ProcessSitemapMonthModels;
+	athena: ProcessSitemapAthena;
+}): Promise<{
+	urls: number;
+}>;
+declare const sitemapUrlStatSchema: v5.LooseObjectSchema<{
+	readonly pk: v5.StringSchema<undefined>;
+	readonly sk: v5.StringSchema<undefined>;
+	readonly path: v5.StringSchema<undefined>;
+	readonly month: v5.StringSchema<undefined>;
+	readonly hits: v5.NumberSchema<undefined>;
+	readonly updatedAt: v5.NumberSchema<undefined>;
+}, undefined>;
+type SitemapUrlStat = v5.InferOutput<typeof sitemapUrlStatSchema>;
+interface SitemapUrlStatRecord {
+	readonly pk: string;
+	readonly sk: string;
+	readonly path: string;
+	readonly month: string;
+	readonly hits: number;
+	readonly updatedAt: number;
+	readonly days: Record<string, number>;
+}
+declare class SitemapUrlStats {
+	private readonly props;
+	constructor(props: {
+		readonly dynamo: Pick<DynamoDBDocumentClient6, "send">;
+		readonly tableName: string;
+		readonly reverseIndexName?: string;
+	});
+	readonly buildAdd: ({ path, month, dayAttr, hits }: {
+		readonly path: string;
+		readonly month: string;
+		readonly dayAttr: string;
+		readonly hits: number;
+	}) => TransactItem;
+	readonly overwriteMany: (stats: Array<SitemapMonthCounts>) => Promise<void>;
+	readonly buildOverwrite: ({ path, month, days, hits }: {
+		readonly path: string;
+		readonly month: string;
+		readonly days: Record<string, number>;
+		readonly hits: number;
+	}) => UpdateMembers3;
+	readonly getForUrl: ({ path, startMonth, endMonth }: {
+		readonly path: string;
+		readonly startMonth: string;
+		readonly endMonth: string;
+	}) => Promise<Array<SitemapUrlStatRecord>>;
+	readonly getForMonth: ({ month }: {
+		readonly month: string;
+	}) => Promise<Array<SitemapUrlStatRecord>>;
+	private readonly parseOne;
+}
 declare function yearKey(date: Date): string;
 declare function monthKey2(date: Date): string;
 declare function dayKey2(date: Date): string;
@@ -455,4 +685,4 @@ declare function hoursBetween({ from, to }: {
 	from: Date;
 	to: Date;
 }): Array<string>;
-export { AnalyticsConfig, AppliedMarker, AppliedMarkers, AppliedSource, AthenaColumnMap, BreakdownInput, BreakdownItem, BreakdownRow, Breakdowns, BucketState, Config, DeviceBucket, Dimension, Grain, HourlyAthenaResult, HourlyAthenaTotals, ProcessHourAthena, ProcessHourModels, ProcessHourOptions, ProcessHourResult, ResolveModels, SummedTotals, Totals, TotalsInput, TotalsItem, TransactItem, View, appliedMarkerSchema, appliedSources, applyDailyRollup, applyMonthlyRollup, breakdownSchema, bucketStates, buildHourlyQuery, chunkTransactItems, classifyBucketState, classifyDevice, columnMaps, configSchema, dayKey2 as dayKey, dimensions, grainForPeriodKey, grains, hourKey2 as hourKey, hourKeyParts, hoursBetween, maxWindowFor, monthKey2 as monthKey, parsePeriodKey, processHour, requiredRecordFields, resolveBreakdownLive, resolveTotalsLive, runHourlyAthenaQuery, shiftAnchor, sumBreakdown, sumTotals, topNWithOther, totalsSchema, windowKeys, yearKey };
+export { AnalyticsConfig, AppliedMarker, AppliedMarkers, AppliedSource, AthenaColumnMap, BreakdownInput, BreakdownItem, BreakdownRow, Breakdowns, BucketState, Config, DeviceBucket, Dimension, Grain, HourlyAthenaResult, HourlyAthenaTotals, MarkerGrain, ProcessHourAthena, ProcessHourModels, ProcessHourOptions, ProcessHourResult, ProcessSitemapAthena, ProcessSitemapModels, ProcessSitemapMonthModels, ProcessSitemapOptions, ResolveModels, SitemapConfig, SitemapHourCounts, SitemapMonthCounts, SitemapUrlStat, SitemapUrlStatRecord, SitemapUrlStats, SummedTotals, Totals, TotalsInput, TotalsItem, TransactItem, View, appliedMarkerSchema, appliedSources, applyDailyRollup, applyMonthlyRollup, breakdownSchema, bucketStates, buildHourlyQuery, buildSitemapHourlyQuery, buildSitemapMonthlyQuery, canonicalPath, chunkTransactItems, classifyBucketState, classifyDevice, columnMaps, configSchema, dayKey2 as dayKey, dimensions, fetchSitemapPaths, grainForPeriodKey, grains, hourKey2 as hourKey, hourKeyParts, hoursBetween, isMarkerRaceError, maxWindowFor, monthKey2 as monthKey, normalizeSitemapLoc, parsePeriodKey, parseSitemap, processHour, processSitemapHour, processSitemapMonth, requiredRecordFields, resolveBreakdownLive, resolveTotalsLive, runHourlyAthenaQuery, runSitemapHourlyQuery, runSitemapMonthlyQuery, sendTransacted, shiftAnchor, sitemapUrlStatSchema, sitemapUserAgent, startAndFetchRows, sumBreakdown, sumTotals, topNWithOther, totalsSchema, windowKeys, yearKey };