@10x-media/analytics 0.1.0-beta.5 → 1.0.0-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/LICENSE +21 -21
- package/README.md +5 -1
- package/dist/adapters/ga4/ga4.js +13 -0
- package/dist/adapters/ga4/ga4.js.map +1 -1
- package/dist/adapters/http/fetchJson.js +1 -1
- package/dist/adapters/plausible/plausible.d.ts +10 -0
- package/dist/adapters/plausible/plausible.js +51 -0
- package/dist/adapters/plausible/plausible.js.map +1 -1
- package/dist/adapters/posthog/posthog.d.ts +10 -1
- package/dist/adapters/posthog/posthog.js +84 -3
- package/dist/adapters/posthog/posthog.js.map +1 -1
- package/dist/adapters/series.js +11 -1
- package/dist/adapters/series.js.map +1 -1
- package/dist/adapters/umami/umami.d.ts +5 -1
- package/dist/adapters/umami/umami.js +28 -0
- package/dist/adapters/umami/umami.js.map +1 -1
- package/dist/capture/getTrackerConfig.d.ts +20 -0
- package/dist/capture/getTrackerConfig.js +31 -0
- package/dist/capture/getTrackerConfig.js.map +1 -0
- package/dist/capture/mountPath.js +14 -0
- package/dist/capture/mountPath.js.map +1 -0
- package/dist/capture/proxyEndpoint.js +123 -0
- package/dist/capture/proxyEndpoint.js.map +1 -0
- package/dist/capture/proxyRoutes.js +108 -0
- package/dist/capture/proxyRoutes.js.map +1 -0
- package/dist/capture/requestBody.js +68 -0
- package/dist/capture/requestBody.js.map +1 -0
- package/dist/capture/slots.d.ts +8 -0
- package/dist/capture/slots.js +43 -0
- package/dist/capture/slots.js.map +1 -0
- package/dist/capture/trackerConfig.d.ts +32 -0
- package/dist/capture/trackerConfig.js +70 -0
- package/dist/capture/trackerConfig.js.map +1 -0
- package/dist/capture/trackerEndpoint.js +35 -0
- package/dist/capture/trackerEndpoint.js.map +1 -0
- package/dist/core/capabilities.d.ts +5 -1
- package/dist/core/capabilities.js +4 -0
- package/dist/core/capabilities.js.map +1 -1
- package/dist/core/capture.d.ts +49 -0
- package/dist/core/contract.d.ts +33 -4
- package/dist/core/contract.js.map +1 -1
- package/dist/core/options.d.ts +97 -2
- package/dist/core/options.js +63 -2
- package/dist/core/options.js.map +1 -1
- package/dist/core/registry.d.ts +1 -0
- package/dist/core/scopeList.js +32 -0
- package/dist/core/scopeList.js.map +1 -0
- package/dist/core/scopedRead.js +13 -6
- package/dist/core/scopedRead.js.map +1 -1
- package/dist/exports/next.d.ts +2 -1
- package/dist/exports/next.js +2 -1
- package/dist/exports/react.d.ts +4 -0
- package/dist/exports/react.js +5 -0
- package/dist/exports/rsc.d.ts +5 -2
- package/dist/exports/rsc.js +3 -1
- package/dist/exports/tracker.d.ts +18 -0
- package/dist/exports/tracker.js +12 -0
- package/dist/exports/types.d.ts +6 -3
- package/dist/fields/AnalyticsPanelClient.js +1 -1
- package/dist/fields/config/narrow.js +2 -0
- package/dist/fields/config/narrow.js.map +1 -1
- package/dist/fields/readForDocument.js +1 -1
- package/dist/goals/match.js +64 -0
- package/dist/goals/match.js.map +1 -0
- package/dist/goals/types.d.ts +33 -0
- package/dist/goals/types.js +7 -0
- package/dist/goals/types.js.map +1 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +39 -10
- package/dist/index.js.map +1 -1
- package/dist/native/collections/events.js +21 -1
- package/dist/native/collections/events.js.map +1 -1
- package/dist/native/collections/rollups.js +39 -3
- package/dist/native/collections/rollups.js.map +1 -1
- package/dist/native/geo/maxmindResolver.js +2 -4
- package/dist/native/geo/maxmindResolver.js.map +1 -1
- package/dist/native/ingest/clientIp.js +15 -0
- package/dist/native/ingest/clientIp.js.map +1 -0
- package/dist/native/ingest/endpoint.js +40 -4
- package/dist/native/ingest/endpoint.js.map +1 -1
- package/dist/native/ingest/flushBatch.js +4 -13
- package/dist/native/ingest/flushBatch.js.map +1 -1
- package/dist/native/ingest/normalizeEvent.js +76 -9
- package/dist/native/ingest/normalizeEvent.js.map +1 -1
- package/dist/native/nativeAdapter.js +100 -5
- package/dist/native/nativeAdapter.js.map +1 -1
- package/dist/native/query/eventAgg.js +161 -0
- package/dist/native/query/eventAgg.js.map +1 -0
- package/dist/native/rollupAcc.js +12 -1
- package/dist/native/rollupAcc.js.map +1 -1
- package/dist/native/rollups/applyDistinctDeltas.js +1 -1
- package/dist/native/rollups/applyDistinctDeltas.js.map +1 -1
- package/dist/native/rollups/bumpRollup.js +7 -2
- package/dist/native/rollups/bumpRollup.js.map +1 -1
- package/dist/native/rollups/deltas.js +70 -11
- package/dist/native/rollups/deltas.js.map +1 -1
- package/dist/next/captureRewrites.d.ts +41 -0
- package/dist/next/captureRewrites.js +56 -0
- package/dist/next/captureRewrites.js.map +1 -0
- package/dist/next/posthogProxyRewrites.d.ts +5 -0
- package/dist/next/posthogProxyRewrites.js +16 -21
- package/dist/next/posthogProxyRewrites.js.map +1 -1
- package/dist/plugin/documentEndpoint.js +1 -1
- package/dist/plugin/paths.js +11 -1
- package/dist/plugin/paths.js.map +1 -1
- package/dist/plugin/runtime.d.ts +1 -0
- package/dist/plugin/runtime.js.map +1 -1
- package/dist/plugin/sourcesEndpoint.js +16 -4
- package/dist/plugin/sourcesEndpoint.js.map +1 -1
- package/dist/plugin/warmTask.js +0 -0
- package/dist/plugin/warmTask.js.map +1 -1
- package/dist/providers/collection.js +56 -16
- package/dist/providers/collection.js.map +1 -1
- package/dist/providers/factory.js +8 -2
- package/dist/providers/factory.js.map +1 -1
- package/dist/react/AnalyticsProvider.d.ts +43 -0
- package/dist/react/AnalyticsProvider.js +66 -0
- package/dist/react/AnalyticsProvider.js.map +1 -0
- package/dist/react/TrackerBoot.d.ts +22 -0
- package/dist/react/TrackerBoot.js +30 -0
- package/dist/react/TrackerBoot.js.map +1 -0
- package/dist/react/useAnalytics.d.ts +17 -0
- package/dist/react/useAnalytics.js +36 -0
- package/dist/react/useAnalytics.js.map +1 -0
- package/dist/rsc/AnalyticsScripts.d.ts +30 -0
- package/dist/rsc/AnalyticsScripts.js +41 -0
- package/dist/rsc/AnalyticsScripts.js.map +1 -0
- package/dist/surfacing/cacheStore.d.ts +1 -0
- package/dist/surfacing/cacheStore.js +18 -2
- package/dist/surfacing/cacheStore.js.map +1 -1
- package/dist/surfacing/engine.d.ts +1 -0
- package/dist/surfacing/engine.js +44 -4
- package/dist/surfacing/engine.js.map +1 -1
- package/dist/surfacing/queue.js +22 -6
- package/dist/surfacing/queue.js.map +1 -1
- package/dist/surfacing/rateLimiter.js +130 -0
- package/dist/surfacing/rateLimiter.js.map +1 -0
- package/dist/surfacing/retryPolicy.js +15 -0
- package/dist/surfacing/retryPolicy.js.map +1 -0
- package/dist/sync/collection.js +23 -7
- package/dist/sync/collection.js.map +1 -1
- package/dist/sync/syncTask.js +68 -33
- package/dist/sync/syncTask.js.map +1 -1
- package/dist/testing/memoryAdapter.d.ts +2 -0
- package/dist/testing/memoryAdapter.js +17 -0
- package/dist/testing/memoryAdapter.js.map +1 -1
- package/dist/tracker/autoCapture.d.ts +48 -0
- package/dist/tracker/autoCapture.js +129 -0
- package/dist/tracker/autoCapture.js.map +1 -0
- package/dist/tracker/consent.d.ts +24 -0
- package/dist/tracker/consent.js +48 -0
- package/dist/tracker/consent.js.map +1 -0
- package/dist/tracker/createTracker.d.ts +18 -0
- package/dist/tracker/createTracker.js +163 -0
- package/dist/tracker/createTracker.js.map +1 -0
- package/dist/tracker/initTracker.d.ts +22 -0
- package/dist/tracker/initTracker.js +33 -0
- package/dist/tracker/initTracker.js.map +1 -0
- package/dist/tracker/loadScript.d.ts +14 -0
- package/dist/tracker/loadScript.js +41 -0
- package/dist/tracker/loadScript.js.map +1 -0
- package/dist/tracker/pageTracking.d.ts +17 -0
- package/dist/tracker/pageTracking.js +38 -0
- package/dist/tracker/pageTracking.js.map +1 -0
- package/dist/tracker/registry.d.ts +27 -0
- package/dist/tracker/registry.js +73 -0
- package/dist/tracker/registry.js.map +1 -0
- package/dist/tracker/sinks/native.d.ts +22 -0
- package/dist/tracker/sinks/native.js +56 -0
- package/dist/tracker/sinks/native.js.map +1 -0
- package/dist/tracker/sinks/plausible.d.ts +14 -0
- package/dist/tracker/sinks/plausible.js +25 -0
- package/dist/tracker/sinks/plausible.js.map +1 -0
- package/dist/tracker/sinks/posthog.d.ts +14 -0
- package/dist/tracker/sinks/posthog.js +18 -0
- package/dist/tracker/sinks/posthog.js.map +1 -0
- package/dist/tracker/sinks/umami.d.ts +13 -0
- package/dist/tracker/sinks/umami.js +17 -0
- package/dist/tracker/sinks/umami.js.map +1 -0
- package/dist/tracker/sinks/vendor.d.ts +14 -0
- package/dist/tracker/sinks/vendor.js +66 -0
- package/dist/tracker/sinks/vendor.js.map +1 -0
- package/dist/tracker/transport.js +28 -0
- package/dist/tracker/transport.js.map +1 -0
- package/dist/tracker/types.d.ts +66 -0
- package/dist/translations/ar.js +83 -0
- package/dist/translations/ar.js.map +1 -0
- package/dist/translations/de.js +6 -0
- package/dist/translations/de.js.map +1 -1
- package/dist/translations/en.js +6 -0
- package/dist/translations/en.js.map +1 -1
- package/dist/translations/es.js +83 -0
- package/dist/translations/es.js.map +1 -0
- package/dist/translations/fr.js +83 -0
- package/dist/translations/fr.js.map +1 -0
- package/dist/translations/id.js +83 -0
- package/dist/translations/id.js.map +1 -0
- package/dist/translations/index.d.ts +10 -1
- package/dist/translations/index.js +19 -1
- package/dist/translations/index.js.map +1 -1
- package/dist/translations/keys.d.ts +6 -0
- package/dist/translations/keys.js +6 -0
- package/dist/translations/keys.js.map +1 -1
- package/dist/translations/ko.js +83 -0
- package/dist/translations/ko.js.map +1 -0
- package/dist/translations/pt.js +83 -0
- package/dist/translations/pt.js.map +1 -0
- package/dist/translations/ru.js +83 -0
- package/dist/translations/ru.js.map +1 -0
- package/dist/translations/uk.js +83 -0
- package/dist/translations/uk.js.map +1 -0
- package/dist/translations/zh.js +83 -0
- package/dist/translations/zh.js.map +1 -0
- package/dist/widgets/breakdownTypes.js +5 -0
- package/dist/widgets/breakdownTypes.js.map +1 -1
- package/dist/widgets/readForWidget.d.ts +2 -1
- package/dist/widgets/readForWidget.js +32 -14
- package/dist/widgets/readForWidget.js.map +1 -1
- package/dist/widgets/readForWidgetBreakdown.d.ts +2 -1
- package/dist/widgets/readForWidgetBreakdown.js +29 -15
- package/dist/widgets/readForWidgetBreakdown.js.map +1 -1
- package/dist/widgets/readForWidgetRealtime.js +1 -1
- package/dist/widgets/readForWidgetSeries.d.ts +2 -1
- package/dist/widgets/readForWidgetSeries.js +34 -16
- package/dist/widgets/readForWidgetSeries.js.map +1 -1
- package/dist/widgets/registerWidgets.js +10 -4
- package/dist/widgets/registerWidgets.js.map +1 -1
- package/dist/widgets/types.js +4 -1
- package/dist/widgets/types.js.map +1 -1
- package/package.json +20 -5
package/dist/surfacing/engine.js
CHANGED
|
@@ -2,6 +2,8 @@ import { startOfDayInTz } from "../timeframe/tz.js";
|
|
|
2
2
|
import { buildCacheKey } from "../core/cacheKey.js";
|
|
3
3
|
import { createCoalescer } from "./coalesce.js";
|
|
4
4
|
import { createQueue } from "./queue.js";
|
|
5
|
+
import { limiterFor } from "./rateLimiter.js";
|
|
6
|
+
import { PROVIDER_READ_TIMEOUT_MESSAGE, shouldRetryProviderError } from "./retryPolicy.js";
|
|
5
7
|
//#region src/surfacing/engine.ts
|
|
6
8
|
const DAY_MS = 864e5;
|
|
7
9
|
const clampRange = (range, maxLookbackDays, tz = "UTC") => {
|
|
@@ -31,7 +33,12 @@ const emptyResult = (provider, q) => ({
|
|
|
31
33
|
});
|
|
32
34
|
function createEngine(opts) {
|
|
33
35
|
const coalesce = createCoalescer();
|
|
34
|
-
const
|
|
36
|
+
const limiters = /* @__PURE__ */ new Map();
|
|
37
|
+
const queue = createQueue({
|
|
38
|
+
...opts.queue,
|
|
39
|
+
shouldRetry: shouldRetryProviderError,
|
|
40
|
+
baseDelayMs: 500
|
|
41
|
+
});
|
|
35
42
|
return { async read(adapter, query) {
|
|
36
43
|
if (!adapter.isConfigured()) return emptyResult(adapter.id, query);
|
|
37
44
|
const { range, clamped } = clampRange(query.dateRange, adapter.capabilities.maxLookbackDays, query.timezone);
|
|
@@ -41,9 +48,42 @@ function createEngine(opts) {
|
|
|
41
48
|
} : query;
|
|
42
49
|
const key = buildCacheKey(adapter.id, q);
|
|
43
50
|
return coalesce(key, async () => {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
const
|
|
51
|
+
let fresh;
|
|
52
|
+
const controller = new AbortController();
|
|
53
|
+
const timer = setTimeout(() => controller.abort(new Error(PROVIDER_READ_TIMEOUT_MESSAGE)), opts.timeoutMs);
|
|
54
|
+
timer.unref?.();
|
|
55
|
+
try {
|
|
56
|
+
const cached = await opts.store.get(key);
|
|
57
|
+
if (cached) return cached;
|
|
58
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
59
|
+
controller.signal.addEventListener("abort", () => reject(controller.signal.reason), { once: true });
|
|
60
|
+
});
|
|
61
|
+
const attempt = queue.run(async () => {
|
|
62
|
+
const release = await limiterFor(limiters, adapter).take(controller.signal);
|
|
63
|
+
try {
|
|
64
|
+
return await adapter.query(q, { signal: controller.signal });
|
|
65
|
+
} finally {
|
|
66
|
+
release();
|
|
67
|
+
}
|
|
68
|
+
}, controller.signal);
|
|
69
|
+
attempt.catch(() => {});
|
|
70
|
+
fresh = await Promise.race([attempt, aborted]);
|
|
71
|
+
} catch (err) {
|
|
72
|
+
opts.onError?.(err, adapter.id);
|
|
73
|
+
try {
|
|
74
|
+
const stale = await opts.store.getStale(key);
|
|
75
|
+
if (stale) return {
|
|
76
|
+
...stale,
|
|
77
|
+
meta: {
|
|
78
|
+
...stale.meta,
|
|
79
|
+
stale: true
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
} catch {}
|
|
83
|
+
throw err;
|
|
84
|
+
} finally {
|
|
85
|
+
clearTimeout(timer);
|
|
86
|
+
}
|
|
47
87
|
const result = clamped ? {
|
|
48
88
|
...fresh,
|
|
49
89
|
meta: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.js","names":[],"sources":["../../src/surfacing/engine.ts"],"sourcesContent":["import { buildCacheKey } from '../core/cacheKey'\nimport type { AnalyticsAdapter, AnalyticsQuery, AnalyticsResult } from '../core/contract'\nimport { DEFAULT_TIMEZONE, startOfDayInTz } from '../timeframe/tz'\nimport type { CacheStore } from './cacheStore'\nimport { createCoalescer } from './coalesce'\nimport { createQueue, type QueueOptions } from './queue'\n\nconst DAY_MS = 86_400_000\n\nconst clampRange = (\n\trange: AnalyticsQuery['dateRange'],\n\tmaxLookbackDays: number | null,\n\ttz: string = DEFAULT_TIMEZONE\n): { range: AnalyticsQuery['dateRange']; clamped: boolean } => {\n\tif (maxLookbackDays == null) {\n\t\treturn { range, clamped: false }\n\t}\n\tconst floor = new Date(startOfDayInTz(range.end, tz).getTime() - maxLookbackDays * DAY_MS)\n\tif (range.start.getTime() >= floor.getTime()) {\n\t\treturn { range, clamped: false }\n\t}\n\treturn { range: { start: floor, end: range.end }, clamped: true }\n}\n\nexport interface EngineOptions {\n\tstore: CacheStore\n\tqueue: QueueOptions\n\t/** Explicit TTL overrides; when a value is unset the adapter's recommendedTtl applies. */\n\tttl: { aggregate?: number; realtime?: number }\n}\n\nexport interface Engine {\n\tread(adapter: AnalyticsAdapter, query: AnalyticsQuery): Promise<AnalyticsResult>\n}\n\nconst emptyResult = (provider: string, q: AnalyticsQuery): AnalyticsResult => ({\n\trows: [],\n\tmeta: { provider, fetchedAt: q.dateRange.end.toISOString() },\n})\n\nexport function createEngine(opts: EngineOptions): Engine {\n\tconst coalesce = createCoalescer<AnalyticsResult>()\n\tconst queue = createQueue(opts.queue)\n\n\treturn {\n\t\tasync read(adapter, query) {\n\t\t\tif (!adapter.isConfigured()) return emptyResult(adapter.id, query)\n\n\t\t\tconst { range, clamped } = clampRange(\n\t\t\t\tquery.dateRange,\n\t\t\t\tadapter.capabilities.maxLookbackDays,\n\t\t\t\tquery.timezone\n\t\t\t)\n\t\t\tconst q = clamped ? { ...query, dateRange: range } : query\n\t\t\tconst key = buildCacheKey(adapter.id, q)\n\t\t\treturn coalesce(key, async () => {\n\t\t\t\tconst cached = await opts.store.get<AnalyticsResult>(key)\n\t\t\t\tif (cached) return cached\n\n\t\t\t\tconst
|
|
1
|
+
{"version":3,"file":"engine.js","names":[],"sources":["../../src/surfacing/engine.ts"],"sourcesContent":["import { buildCacheKey } from '../core/cacheKey'\nimport type { AnalyticsAdapter, AnalyticsQuery, AnalyticsResult } from '../core/contract'\nimport { DEFAULT_TIMEZONE, startOfDayInTz } from '../timeframe/tz'\nimport type { CacheStore } from './cacheStore'\nimport { createCoalescer } from './coalesce'\nimport { createQueue, type QueueOptions } from './queue'\nimport { limiterFor, type RateLimiter } from './rateLimiter'\nimport { PROVIDER_READ_TIMEOUT_MESSAGE, shouldRetryProviderError } from './retryPolicy'\n\nconst DAY_MS = 86_400_000\n\nconst clampRange = (\n\trange: AnalyticsQuery['dateRange'],\n\tmaxLookbackDays: number | null,\n\ttz: string = DEFAULT_TIMEZONE\n): { range: AnalyticsQuery['dateRange']; clamped: boolean } => {\n\tif (maxLookbackDays == null) {\n\t\treturn { range, clamped: false }\n\t}\n\tconst floor = new Date(startOfDayInTz(range.end, tz).getTime() - maxLookbackDays * DAY_MS)\n\tif (range.start.getTime() >= floor.getTime()) {\n\t\treturn { range, clamped: false }\n\t}\n\treturn { range: { start: floor, end: range.end }, clamped: true }\n}\n\nexport interface EngineOptions {\n\tstore: CacheStore\n\tqueue: QueueOptions\n\t/** Explicit TTL overrides; when a value is unset the adapter's recommendedTtl applies. */\n\tttl: { aggregate?: number; realtime?: number }\n\t/** Budget for a single adapter read; the in-flight request is aborted past this. */\n\ttimeoutMs: number\n\t/** Called once per failed adapter fetch, before falling back to a stale cache entry. */\n\tonError?: (err: unknown, adapterId: string) => void\n}\n\nexport interface Engine {\n\tread(adapter: AnalyticsAdapter, query: AnalyticsQuery): Promise<AnalyticsResult>\n}\n\nconst emptyResult = (provider: string, q: AnalyticsQuery): AnalyticsResult => ({\n\trows: [],\n\tmeta: { provider, fetchedAt: q.dateRange.end.toISOString() },\n})\n\nexport function createEngine(opts: EngineOptions): Engine {\n\tconst coalesce = createCoalescer<AnalyticsResult>()\n\tconst limiters = new Map<string, RateLimiter>()\n\tconst queue = createQueue({\n\t\t...opts.queue,\n\t\tshouldRetry: shouldRetryProviderError,\n\t\tbaseDelayMs: 500,\n\t})\n\n\treturn {\n\t\tasync read(adapter, query) {\n\t\t\tif (!adapter.isConfigured()) return emptyResult(adapter.id, query)\n\n\t\t\tconst { range, clamped } = clampRange(\n\t\t\t\tquery.dateRange,\n\t\t\t\tadapter.capabilities.maxLookbackDays,\n\t\t\t\tquery.timezone\n\t\t\t)\n\t\t\tconst q = clamped ? { ...query, dateRange: range } : query\n\t\t\tconst key = buildCacheKey(adapter.id, q)\n\t\t\treturn coalesce(key, async () => {\n\t\t\t\tlet fresh: AnalyticsResult\n\t\t\t\tconst controller = new AbortController()\n\t\t\t\tconst timer = setTimeout(\n\t\t\t\t\t() => controller.abort(new Error(PROVIDER_READ_TIMEOUT_MESSAGE)),\n\t\t\t\t\topts.timeoutMs\n\t\t\t\t)\n\t\t\t\ttimer.unref?.()\n\t\t\t\ttry {\n\t\t\t\t\tconst cached = await opts.store.get<AnalyticsResult>(key)\n\t\t\t\t\tif (cached) return cached\n\n\t\t\t\t\t// Races the queued attempt itself, not just its backoff waits, so a signal-blind\n\t\t\t\t\t// adapter (one that never checks ctx.signal) still bounds the read at timeoutMs.\n\t\t\t\t\t// The orphaned attempt keeps its queue slot until the underlying client gives up;\n\t\t\t\t\t// this only bounds how long read() waits for it.\n\t\t\t\t\tconst aborted = new Promise<never>((_resolve, reject) => {\n\t\t\t\t\t\tcontroller.signal.addEventListener('abort', () => reject(controller.signal.reason), {\n\t\t\t\t\t\t\tonce: true,\n\t\t\t\t\t\t})\n\t\t\t\t\t})\n\t\t\t\t\tconst attempt = queue.run(async () => {\n\t\t\t\t\t\tconst release = await limiterFor(limiters, adapter).take(controller.signal)\n\t\t\t\t\t\ttry {\n\t\t\t\t\t\t\treturn await adapter.query(q, { signal: controller.signal })\n\t\t\t\t\t\t} finally {\n\t\t\t\t\t\t\trelease()\n\t\t\t\t\t\t}\n\t\t\t\t\t}, controller.signal)\n\t\t\t\t\tattempt.catch(() => {})\n\t\t\t\t\tfresh = await Promise.race([attempt, aborted])\n\t\t\t\t} catch (err) {\n\t\t\t\t\topts.onError?.(err, adapter.id)\n\t\t\t\t\ttry {\n\t\t\t\t\t\tconst stale = await opts.store.getStale<AnalyticsResult>(key)\n\t\t\t\t\t\tif (stale) return { ...stale, meta: { ...stale.meta, stale: true } }\n\t\t\t\t\t} catch {\n\t\t\t\t\t\t// A broken store fails getStale too; surface the original read error, not this one.\n\t\t\t\t\t}\n\t\t\t\t\tthrow err\n\t\t\t\t} finally {\n\t\t\t\t\tclearTimeout(timer)\n\t\t\t\t}\n\t\t\t\tconst result: AnalyticsResult = clamped\n\t\t\t\t\t? { ...fresh, meta: { ...fresh.meta, clamped: true } }\n\t\t\t\t\t: fresh\n\t\t\t\tconst ttl = opts.ttl.aggregate ?? adapter.capabilities.recommendedTtl.aggregate\n\t\t\t\tawait opts.store.set(key, result, ttl)\n\t\t\t\treturn result\n\t\t\t})\n\t\t},\n\t}\n}\n"],"mappings":";;;;;;;AASA,MAAM,SAAS;AAEf,MAAM,cACL,OACA,iBACA,KAAA,UAC8D;CAC9D,IAAI,mBAAmB,MACtB,OAAO;EAAE;EAAO,SAAS;CAAM;CAEhC,MAAM,wBAAQ,IAAI,KAAK,eAAe,MAAM,KAAK,EAAE,EAAE,QAAQ,IAAI,kBAAkB,MAAM;CACzF,IAAI,MAAM,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAC1C,OAAO;EAAE;EAAO,SAAS;CAAM;CAEhC,OAAO;EAAE,OAAO;GAAE,OAAO;GAAO,KAAK,MAAM;EAAI;EAAG,SAAS;CAAK;AACjE;AAiBA,MAAM,eAAe,UAAkB,OAAwC;CAC9E,MAAM,CAAC;CACP,MAAM;EAAE;EAAU,WAAW,EAAE,UAAU,IAAI,YAAY;CAAE;AAC5D;AAEA,SAAgB,aAAa,MAA6B;CACzD,MAAM,WAAW,gBAAiC;CAClD,MAAM,2BAAW,IAAI,IAAyB;CAC9C,MAAM,QAAQ,YAAY;EACzB,GAAG,KAAK;EACR,aAAa;EACb,aAAa;CACd,CAAC;CAED,OAAO,EACN,MAAM,KAAK,SAAS,OAAO;EAC1B,IAAI,CAAC,QAAQ,aAAa,GAAG,OAAO,YAAY,QAAQ,IAAI,KAAK;EAEjE,MAAM,EAAE,OAAO,YAAY,WAC1B,MAAM,WACN,QAAQ,aAAa,iBACrB,MAAM,QACP;EACA,MAAM,IAAI,UAAU;GAAE,GAAG;GAAO,WAAW;EAAM,IAAI;EACrD,MAAM,MAAM,cAAc,QAAQ,IAAI,CAAC;EACvC,OAAO,SAAS,KAAK,YAAY;GAChC,IAAI;GACJ,MAAM,aAAa,IAAI,gBAAgB;GACvC,MAAM,QAAQ,iBACP,WAAW,MAAM,IAAI,MAAM,6BAA6B,CAAC,GAC/D,KAAK,SACN;GACA,MAAM,QAAQ;GACd,IAAI;IACH,MAAM,SAAS,MAAM,KAAK,MAAM,IAAqB,GAAG;IACxD,IAAI,QAAQ,OAAO;IAMnB,MAAM,UAAU,IAAI,SAAgB,UAAU,WAAW;KACxD,WAAW,OAAO,iBAAiB,eAAe,OAAO,WAAW,OAAO,MAAM,GAAG,EACnF,MAAM,KACP,CAAC;IACF,CAAC;IACD,MAAM,UAAU,MAAM,IAAI,YAAY;KACrC,MAAM,UAAU,MAAM,WAAW,UAAU,OAAO,EAAE,KAAK,WAAW,MAAM;KAC1E,IAAI;MACH,OAAO,MAAM,QAAQ,MAAM,GAAG,EAAE,QAAQ,WAAW,OAAO,CAAC;KAC5D,UAAU;MACT,QAAQ;KACT;IACD,GAAG,WAAW,MAAM;IACpB,QAAQ,YAAY,CAAC,CAAC;IACtB,QAAQ,MAAM,QAAQ,KAAK,CAAC,SAAS,OAAO,CAAC;GAC9C,SAAS,KAAK;IACb,KAAK,UAAU,KAAK,QAAQ,EAAE;IAC9B,IAAI;KACH,MAAM,QAAQ,MAAM,KAAK,MAAM,SAA0B,GAAG;KAC5D,IAAI,OAAO,OAAO;MAAE,GAAG;MAAO,MAAM;OAAE,GAAG,MAAM;OAAM,OAAO;MAAK;KAAE;IACpE,QAAQ,CAER;IACA,MAAM;GACP,UAAU;IACT,aAAa,KAAK;GACnB;GACA,MAAM,SAA0B,UAC7B;IAAE,GAAG;IAAO,MAAM;KAAE,GAAG,MAAM;KAAM,SAAS;IAAK;GAAE,IACnD;GACH,MAAM,MAAM,KAAK,IAAI,aAAa,QAAQ,aAAa,eAAe;GACtE,MAAM,KAAK,MAAM,IAAI,KAAK,QAAQ,GAAG;GACrC,OAAO;EACR,CAAC;CACF,EACD;AACD"}
|
package/dist/surfacing/queue.js
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
//#region src/surfacing/queue.ts
|
|
2
|
-
|
|
2
|
+
/** Resolves after ms, or rethrows the original task error early if signal aborts first. */
|
|
3
|
+
const wait = (ms, taskErr, signal) => {
|
|
4
|
+
if (!signal) return new Promise((r) => setTimeout(r, ms));
|
|
5
|
+
if (signal.aborted) return Promise.reject(taskErr);
|
|
6
|
+
return new Promise((resolve, reject) => {
|
|
7
|
+
const timer = setTimeout(() => {
|
|
8
|
+
signal.removeEventListener("abort", onAbort);
|
|
9
|
+
resolve();
|
|
10
|
+
}, ms);
|
|
11
|
+
const onAbort = () => {
|
|
12
|
+
clearTimeout(timer);
|
|
13
|
+
reject(taskErr);
|
|
14
|
+
};
|
|
15
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
16
|
+
});
|
|
17
|
+
};
|
|
3
18
|
function createQueue(opts) {
|
|
4
19
|
const maxRetries = opts.maxRetries ?? 0;
|
|
5
20
|
const baseDelayMs = opts.baseDelayMs ?? 200;
|
|
@@ -18,21 +33,22 @@ function createQueue(opts) {
|
|
|
18
33
|
active--;
|
|
19
34
|
pending.shift()?.();
|
|
20
35
|
};
|
|
21
|
-
const withRetry = async (task) => {
|
|
36
|
+
const withRetry = async (task, signal) => {
|
|
22
37
|
let attempt = 0;
|
|
23
38
|
for (;;) try {
|
|
24
39
|
return await task();
|
|
25
40
|
} catch (err) {
|
|
26
|
-
if (attempt
|
|
41
|
+
if (!(opts.shouldRetry ? opts.shouldRetry(err, attempt) : attempt < maxRetries)) throw err;
|
|
27
42
|
const jitter = baseDelayMs * .5;
|
|
28
|
-
await wait(baseDelayMs * 2 ** attempt + jitter * attempt);
|
|
43
|
+
await wait(baseDelayMs * 2 ** attempt + jitter * attempt, err, signal);
|
|
29
44
|
attempt++;
|
|
30
45
|
}
|
|
31
46
|
};
|
|
32
|
-
return { async run(task) {
|
|
47
|
+
return { async run(task, signal) {
|
|
48
|
+
if (signal?.aborted) throw signal.reason;
|
|
33
49
|
await acquire();
|
|
34
50
|
try {
|
|
35
|
-
return await withRetry(task);
|
|
51
|
+
return await withRetry(task, signal);
|
|
36
52
|
} finally {
|
|
37
53
|
release();
|
|
38
54
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.js","names":[],"sources":["../../src/surfacing/queue.ts"],"sourcesContent":["export interface QueueOptions {\n\tconcurrency: number\n\tmaxRetries?: number\n\tbaseDelayMs?: number\n}\n\nexport interface Queue {\n\trun<T>(task: () => Promise<T
|
|
1
|
+
{"version":3,"file":"queue.js","names":[],"sources":["../../src/surfacing/queue.ts"],"sourcesContent":["export interface QueueOptions {\n\tconcurrency: number\n\tmaxRetries?: number\n\tbaseDelayMs?: number\n\t/** Consulted per attempt in place of the maxRetries check when provided. */\n\tshouldRetry?: (err: unknown, attempt: number) => boolean\n}\n\nexport interface Queue {\n\trun<T>(task: () => Promise<T>, signal?: AbortSignal): Promise<T>\n}\n\n/** Resolves after ms, or rethrows the original task error early if signal aborts first. */\nconst wait = (ms: number, taskErr: unknown, signal?: AbortSignal): Promise<void> => {\n\tif (!signal) return new Promise((r) => setTimeout(r, ms))\n\t// An abort that already fired won't fire its event again, so a listener added\n\t// after the fact never triggers; check the already-aborted case up front.\n\tif (signal.aborted) return Promise.reject(taskErr)\n\treturn new Promise<void>((resolve, reject) => {\n\t\tconst timer = setTimeout(() => {\n\t\t\tsignal.removeEventListener('abort', onAbort)\n\t\t\tresolve()\n\t\t}, ms)\n\t\tconst onAbort = () => {\n\t\t\tclearTimeout(timer)\n\t\t\treject(taskErr)\n\t\t}\n\t\tsignal.addEventListener('abort', onAbort, { once: true })\n\t})\n}\n\nexport function createQueue(opts: QueueOptions): Queue {\n\tconst maxRetries = opts.maxRetries ?? 0\n\tconst baseDelayMs = opts.baseDelayMs ?? 200\n\tlet active = 0\n\tconst pending: Array<() => void> = []\n\n\tconst acquire = (): Promise<void> => {\n\t\tif (active < opts.concurrency) {\n\t\t\tactive++\n\t\t\treturn Promise.resolve()\n\t\t}\n\t\treturn new Promise<void>((resolve) => pending.push(resolve)).then(() => {\n\t\t\tactive++\n\t\t})\n\t}\n\n\tconst release = (): void => {\n\t\tactive--\n\t\tpending.shift()?.()\n\t}\n\n\tconst withRetry = async <T>(task: () => Promise<T>, signal?: AbortSignal): Promise<T> => {\n\t\tlet attempt = 0\n\t\tfor (;;) {\n\t\t\ttry {\n\t\t\t\treturn await task()\n\t\t\t} catch (err) {\n\t\t\t\tconst canRetry = opts.shouldRetry ? opts.shouldRetry(err, attempt) : attempt < maxRetries\n\t\t\t\tif (!canRetry) throw err\n\t\t\t\t// Jitter derived from attempt count, not Math.random, to keep tests deterministic\n\t\t\t\tconst jitter = baseDelayMs * 0.5\n\t\t\t\tawait wait(baseDelayMs * 2 ** attempt + jitter * attempt, err, signal)\n\t\t\t\tattempt++\n\t\t\t}\n\t\t}\n\t}\n\n\treturn {\n\t\tasync run<T>(task: () => Promise<T>, signal?: AbortSignal): Promise<T> {\n\t\t\tif (signal?.aborted) throw signal.reason\n\t\t\tawait acquire()\n\t\t\ttry {\n\t\t\t\treturn await withRetry(task, signal)\n\t\t\t} finally {\n\t\t\t\trelease()\n\t\t\t}\n\t\t},\n\t}\n}\n"],"mappings":";;AAaA,MAAM,QAAQ,IAAY,SAAkB,WAAwC;CACnF,IAAI,CAAC,QAAQ,OAAO,IAAI,SAAS,MAAM,WAAW,GAAG,EAAE,CAAC;CAGxD,IAAI,OAAO,SAAS,OAAO,QAAQ,OAAO,OAAO;CACjD,OAAO,IAAI,SAAe,SAAS,WAAW;EAC7C,MAAM,QAAQ,iBAAiB;GAC9B,OAAO,oBAAoB,SAAS,OAAO;GAC3C,QAAQ;EACT,GAAG,EAAE;EACL,MAAM,gBAAgB;GACrB,aAAa,KAAK;GAClB,OAAO,OAAO;EACf;EACA,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CACzD,CAAC;AACF;AAEA,SAAgB,YAAY,MAA2B;CACtD,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,cAAc,KAAK,eAAe;CACxC,IAAI,SAAS;CACb,MAAM,UAA6B,CAAC;CAEpC,MAAM,gBAA+B;EACpC,IAAI,SAAS,KAAK,aAAa;GAC9B;GACA,OAAO,QAAQ,QAAQ;EACxB;EACA,OAAO,IAAI,SAAe,YAAY,QAAQ,KAAK,OAAO,CAAC,EAAE,WAAW;GACvE;EACD,CAAC;CACF;CAEA,MAAM,gBAAsB;EAC3B;EACA,QAAQ,MAAM,IAAI;CACnB;CAEA,MAAM,YAAY,OAAU,MAAwB,WAAqC;EACxF,IAAI,UAAU;EACd,SACC,IAAI;GACH,OAAO,MAAM,KAAK;EACnB,SAAS,KAAK;GAEb,IAAI,EADa,KAAK,cAAc,KAAK,YAAY,KAAK,OAAO,IAAI,UAAU,aAChE,MAAM;GAErB,MAAM,SAAS,cAAc;GAC7B,MAAM,KAAK,cAAc,KAAK,UAAU,SAAS,SAAS,KAAK,MAAM;GACrE;EACD;CAEF;CAEA,OAAO,EACN,MAAM,IAAO,MAAwB,QAAkC;EACtE,IAAI,QAAQ,SAAS,MAAM,OAAO;EAClC,MAAM,QAAQ;EACd,IAAI;GACH,OAAO,MAAM,UAAU,MAAM,MAAM;EACpC,UAAU;GACT,QAAQ;EACT;CACD,EACD;AACD"}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
//#region src/surfacing/rateLimiter.ts
|
|
2
|
+
const MINUTE_MS = 6e4;
|
|
3
|
+
const HOUR_MS = 36e5;
|
|
4
|
+
const createBucket = (capacity, windowMs) => {
|
|
5
|
+
const intervalMs = windowMs / capacity;
|
|
6
|
+
let tokens = capacity;
|
|
7
|
+
let last;
|
|
8
|
+
const sync = (now) => {
|
|
9
|
+
if (last === void 0) {
|
|
10
|
+
last = now;
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
const elapsed = now - last;
|
|
14
|
+
if (elapsed > 0) {
|
|
15
|
+
tokens = Math.min(capacity, tokens + elapsed / intervalMs);
|
|
16
|
+
last = now;
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
return {
|
|
20
|
+
waitMs(now) {
|
|
21
|
+
sync(now);
|
|
22
|
+
if (tokens >= 1) return 0;
|
|
23
|
+
return Math.ceil((1 - tokens) * intervalMs);
|
|
24
|
+
},
|
|
25
|
+
take(now) {
|
|
26
|
+
sync(now);
|
|
27
|
+
tokens -= 1;
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
const sleep = (ms, signal) => {
|
|
32
|
+
if (signal?.aborted) return Promise.reject(signal.reason);
|
|
33
|
+
return new Promise((resolve, reject) => {
|
|
34
|
+
const timer = setTimeout(() => {
|
|
35
|
+
signal?.removeEventListener("abort", onAbort);
|
|
36
|
+
resolve();
|
|
37
|
+
}, ms);
|
|
38
|
+
const onAbort = () => {
|
|
39
|
+
clearTimeout(timer);
|
|
40
|
+
reject(signal?.reason);
|
|
41
|
+
};
|
|
42
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
43
|
+
});
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* Waits until every bucket holds a token, then deducts one from each. Bucket checks and
|
|
47
|
+
* deductions happen synchronously (no await between them) so a zero-wait pass can never
|
|
48
|
+
* race; a non-zero wait rechecks after sleeping since a concurrent taker may have spent
|
|
49
|
+
* the token first.
|
|
50
|
+
*/
|
|
51
|
+
const acquireTokens = async (buckets, clock, signal) => {
|
|
52
|
+
for (;;) {
|
|
53
|
+
if (signal?.aborted) throw signal.reason;
|
|
54
|
+
const now = clock();
|
|
55
|
+
const maxWait = buckets.reduce((max, bucket) => Math.max(max, bucket.waitMs(now)), 0);
|
|
56
|
+
if (maxWait === 0) {
|
|
57
|
+
for (const bucket of buckets) bucket.take(now);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
await sleep(maxWait, signal);
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
const createSemaphore = (capacity) => {
|
|
64
|
+
let active = 0;
|
|
65
|
+
const waiters = [];
|
|
66
|
+
const release = () => {
|
|
67
|
+
active--;
|
|
68
|
+
waiters.shift()?.();
|
|
69
|
+
};
|
|
70
|
+
return { acquire(signal) {
|
|
71
|
+
if (signal?.aborted) return Promise.reject(signal.reason);
|
|
72
|
+
if (active < capacity) {
|
|
73
|
+
active++;
|
|
74
|
+
return Promise.resolve(release);
|
|
75
|
+
}
|
|
76
|
+
return new Promise((resolve, reject) => {
|
|
77
|
+
const onAbort = () => {
|
|
78
|
+
const idx = waiters.indexOf(grant);
|
|
79
|
+
if (idx !== -1) waiters.splice(idx, 1);
|
|
80
|
+
reject(signal?.reason);
|
|
81
|
+
};
|
|
82
|
+
const grant = () => {
|
|
83
|
+
signal?.removeEventListener("abort", onAbort);
|
|
84
|
+
active++;
|
|
85
|
+
resolve(release);
|
|
86
|
+
};
|
|
87
|
+
waiters.push(grant);
|
|
88
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
89
|
+
});
|
|
90
|
+
} };
|
|
91
|
+
};
|
|
92
|
+
const noopRelease = () => {};
|
|
93
|
+
/**
|
|
94
|
+
* Builds a limiter from an adapter's declared RateLimitDescriptor; a null descriptor
|
|
95
|
+
* returns an unlimited pass-through. requestsPerMinute and requestsPerHour each maintain
|
|
96
|
+
* their own continuously refilling token bucket and a take needs a token from every
|
|
97
|
+
* configured bucket; maxConcurrent is a plain semaphore released by the returned function.
|
|
98
|
+
* Token deductions are never released; they represent quota already spent.
|
|
99
|
+
*/
|
|
100
|
+
const createRateLimiter = (descriptor, clock = Date.now) => {
|
|
101
|
+
if (!descriptor) return { take: async () => noopRelease };
|
|
102
|
+
const buckets = [];
|
|
103
|
+
if (descriptor.requestsPerMinute) buckets.push(createBucket(descriptor.requestsPerMinute, MINUTE_MS));
|
|
104
|
+
if (descriptor.requestsPerHour) buckets.push(createBucket(descriptor.requestsPerHour, HOUR_MS));
|
|
105
|
+
const semaphore = descriptor.maxConcurrent ? createSemaphore(descriptor.maxConcurrent) : void 0;
|
|
106
|
+
return { async take(signal) {
|
|
107
|
+
if (signal?.aborted) throw signal.reason;
|
|
108
|
+
const releaseConcurrency = semaphore ? await semaphore.acquire(signal) : noopRelease;
|
|
109
|
+
try {
|
|
110
|
+
if (buckets.length) await acquireTokens(buckets, clock, signal);
|
|
111
|
+
return releaseConcurrency;
|
|
112
|
+
} catch (err) {
|
|
113
|
+
releaseConcurrency();
|
|
114
|
+
throw err;
|
|
115
|
+
}
|
|
116
|
+
} };
|
|
117
|
+
};
|
|
118
|
+
/** Returns the adapter's limiter, building and caching it from its declared rateLimit on first use. */
|
|
119
|
+
const limiterFor = (map, adapter, clock) => {
|
|
120
|
+
let limiter = map.get(adapter.id);
|
|
121
|
+
if (!limiter) {
|
|
122
|
+
limiter = createRateLimiter(adapter.capabilities.rateLimit, clock);
|
|
123
|
+
map.set(adapter.id, limiter);
|
|
124
|
+
}
|
|
125
|
+
return limiter;
|
|
126
|
+
};
|
|
127
|
+
//#endregion
|
|
128
|
+
export { limiterFor };
|
|
129
|
+
|
|
130
|
+
//# sourceMappingURL=rateLimiter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rateLimiter.js","names":[],"sources":["../../src/surfacing/rateLimiter.ts"],"sourcesContent":["import type { AnalyticsAdapter, RateLimitDescriptor } from '../core/contract'\n\nexport interface RateLimiter {\n\t/** Resolves when a request slot is available; rejects with the signal's reason when aborted while waiting. Release must be called exactly once per successful take. */\n\ttake(signal?: AbortSignal): Promise<() => void>\n}\n\nconst MINUTE_MS = 60_000\nconst HOUR_MS = 3_600_000\n\ninterface TokenBucket {\n\twaitMs(now: number): number\n\ttake(now: number): void\n}\n\nconst createBucket = (capacity: number, windowMs: number): TokenBucket => {\n\t// ms per token, rather than tokens-per-ms: for every descriptor value in this codebase\n\t// windowMs divides evenly by capacity, so this stays exact and sidesteps the float\n\t// drift a capacity/windowMs rate would reintroduce on the multiply-then-divide round trip.\n\tconst intervalMs = windowMs / capacity\n\tlet tokens = capacity\n\tlet last: number | undefined\n\n\tconst sync = (now: number): void => {\n\t\tif (last === undefined) {\n\t\t\tlast = now\n\t\t\treturn\n\t\t}\n\t\tconst elapsed = now - last\n\t\tif (elapsed > 0) {\n\t\t\ttokens = Math.min(capacity, tokens + elapsed / intervalMs)\n\t\t\tlast = now\n\t\t}\n\t}\n\n\treturn {\n\t\twaitMs(now) {\n\t\t\tsync(now)\n\t\t\tif (tokens >= 1) return 0\n\t\t\treturn Math.ceil((1 - tokens) * intervalMs)\n\t\t},\n\t\ttake(now) {\n\t\t\tsync(now)\n\t\t\ttokens -= 1\n\t\t},\n\t}\n}\n\nconst sleep = (ms: number, signal?: AbortSignal): Promise<void> => {\n\tif (signal?.aborted) return Promise.reject(signal.reason)\n\treturn new Promise<void>((resolve, reject) => {\n\t\tconst timer = setTimeout(() => {\n\t\t\tsignal?.removeEventListener('abort', onAbort)\n\t\t\tresolve()\n\t\t}, ms)\n\t\tconst onAbort = (): void => {\n\t\t\tclearTimeout(timer)\n\t\t\treject(signal?.reason)\n\t\t}\n\t\tsignal?.addEventListener('abort', onAbort, { once: true })\n\t})\n}\n\n/**\n * Waits until every bucket holds a token, then deducts one from each. Bucket checks and\n * deductions happen synchronously (no await between them) so a zero-wait pass can never\n * race; a non-zero wait rechecks after sleeping since a concurrent taker may have spent\n * the token first.\n */\nconst acquireTokens = async (\n\tbuckets: TokenBucket[],\n\tclock: () => number,\n\tsignal?: AbortSignal\n): Promise<void> => {\n\tfor (;;) {\n\t\tif (signal?.aborted) throw signal.reason\n\t\tconst now = clock()\n\t\tconst maxWait = buckets.reduce((max, bucket) => Math.max(max, bucket.waitMs(now)), 0)\n\t\tif (maxWait === 0) {\n\t\t\tfor (const bucket of buckets) bucket.take(now)\n\t\t\treturn\n\t\t}\n\t\tawait sleep(maxWait, signal)\n\t}\n}\n\ninterface Semaphore {\n\tacquire(signal?: AbortSignal): Promise<() => void>\n}\n\nconst createSemaphore = (capacity: number): Semaphore => {\n\tlet active = 0\n\tconst waiters: Array<() => void> = []\n\n\tconst release = (): void => {\n\t\tactive--\n\t\twaiters.shift()?.()\n\t}\n\n\treturn {\n\t\tacquire(signal) {\n\t\t\tif (signal?.aborted) return Promise.reject(signal.reason)\n\t\t\tif (active < capacity) {\n\t\t\t\tactive++\n\t\t\t\treturn Promise.resolve(release)\n\t\t\t}\n\t\t\treturn new Promise<() => void>((resolve, reject) => {\n\t\t\t\tconst onAbort = (): void => {\n\t\t\t\t\tconst idx = waiters.indexOf(grant)\n\t\t\t\t\tif (idx !== -1) waiters.splice(idx, 1)\n\t\t\t\t\treject(signal?.reason)\n\t\t\t\t}\n\t\t\t\tconst grant = (): void => {\n\t\t\t\t\tsignal?.removeEventListener('abort', onAbort)\n\t\t\t\t\tactive++\n\t\t\t\t\tresolve(release)\n\t\t\t\t}\n\t\t\t\twaiters.push(grant)\n\t\t\t\tsignal?.addEventListener('abort', onAbort, { once: true })\n\t\t\t})\n\t\t},\n\t}\n}\n\nconst noopRelease = (): void => {}\n\n/**\n * Builds a limiter from an adapter's declared RateLimitDescriptor; a null descriptor\n * returns an unlimited pass-through. requestsPerMinute and requestsPerHour each maintain\n * their own continuously refilling token bucket and a take needs a token from every\n * configured bucket; maxConcurrent is a plain semaphore released by the returned function.\n * Token deductions are never released; they represent quota already spent.\n */\nexport const createRateLimiter = (\n\tdescriptor: RateLimitDescriptor | null,\n\tclock: () => number = Date.now\n): RateLimiter => {\n\tif (!descriptor) {\n\t\treturn { take: async () => noopRelease }\n\t}\n\n\tconst buckets: TokenBucket[] = []\n\tif (descriptor.requestsPerMinute)\n\t\tbuckets.push(createBucket(descriptor.requestsPerMinute, MINUTE_MS))\n\tif (descriptor.requestsPerHour) buckets.push(createBucket(descriptor.requestsPerHour, HOUR_MS))\n\tconst semaphore = descriptor.maxConcurrent ? createSemaphore(descriptor.maxConcurrent) : undefined\n\n\treturn {\n\t\tasync take(signal) {\n\t\t\tif (signal?.aborted) throw signal.reason\n\t\t\tconst releaseConcurrency = semaphore ? await semaphore.acquire(signal) : noopRelease\n\t\t\ttry {\n\t\t\t\tif (buckets.length) await acquireTokens(buckets, clock, signal)\n\t\t\t\treturn releaseConcurrency\n\t\t\t} catch (err) {\n\t\t\t\treleaseConcurrency()\n\t\t\t\tthrow err\n\t\t\t}\n\t\t},\n\t}\n}\n\n/** Returns the adapter's limiter, building and caching it from its declared rateLimit on first use. */\nexport const limiterFor = (\n\tmap: Map<string, RateLimiter>,\n\tadapter: AnalyticsAdapter,\n\tclock?: () => number\n): RateLimiter => {\n\tlet limiter = map.get(adapter.id)\n\tif (!limiter) {\n\t\tlimiter = createRateLimiter(adapter.capabilities.rateLimit, clock)\n\t\tmap.set(adapter.id, limiter)\n\t}\n\treturn limiter\n}\n"],"mappings":";AAOA,MAAM,YAAY;AAClB,MAAM,UAAU;AAOhB,MAAM,gBAAgB,UAAkB,aAAkC;CAIzE,MAAM,aAAa,WAAW;CAC9B,IAAI,SAAS;CACb,IAAI;CAEJ,MAAM,QAAQ,QAAsB;EACnC,IAAI,SAAS,KAAA,GAAW;GACvB,OAAO;GACP;EACD;EACA,MAAM,UAAU,MAAM;EACtB,IAAI,UAAU,GAAG;GAChB,SAAS,KAAK,IAAI,UAAU,SAAS,UAAU,UAAU;GACzD,OAAO;EACR;CACD;CAEA,OAAO;EACN,OAAO,KAAK;GACX,KAAK,GAAG;GACR,IAAI,UAAU,GAAG,OAAO;GACxB,OAAO,KAAK,MAAM,IAAI,UAAU,UAAU;EAC3C;EACA,KAAK,KAAK;GACT,KAAK,GAAG;GACR,UAAU;EACX;CACD;AACD;AAEA,MAAM,SAAS,IAAY,WAAwC;CAClE,IAAI,QAAQ,SAAS,OAAO,QAAQ,OAAO,OAAO,MAAM;CACxD,OAAO,IAAI,SAAe,SAAS,WAAW;EAC7C,MAAM,QAAQ,iBAAiB;GAC9B,QAAQ,oBAAoB,SAAS,OAAO;GAC5C,QAAQ;EACT,GAAG,EAAE;EACL,MAAM,gBAAsB;GAC3B,aAAa,KAAK;GAClB,OAAO,QAAQ,MAAM;EACtB;EACA,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CAC1D,CAAC;AACF;;;;;;;AAQA,MAAM,gBAAgB,OACrB,SACA,OACA,WACmB;CACnB,SAAS;EACR,IAAI,QAAQ,SAAS,MAAM,OAAO;EAClC,MAAM,MAAM,MAAM;EAClB,MAAM,UAAU,QAAQ,QAAQ,KAAK,WAAW,KAAK,IAAI,KAAK,OAAO,OAAO,GAAG,CAAC,GAAG,CAAC;EACpF,IAAI,YAAY,GAAG;GAClB,KAAK,MAAM,UAAU,SAAS,OAAO,KAAK,GAAG;GAC7C;EACD;EACA,MAAM,MAAM,SAAS,MAAM;CAC5B;AACD;AAMA,MAAM,mBAAmB,aAAgC;CACxD,IAAI,SAAS;CACb,MAAM,UAA6B,CAAC;CAEpC,MAAM,gBAAsB;EAC3B;EACA,QAAQ,MAAM,IAAI;CACnB;CAEA,OAAO,EACN,QAAQ,QAAQ;EACf,IAAI,QAAQ,SAAS,OAAO,QAAQ,OAAO,OAAO,MAAM;EACxD,IAAI,SAAS,UAAU;GACtB;GACA,OAAO,QAAQ,QAAQ,OAAO;EAC/B;EACA,OAAO,IAAI,SAAqB,SAAS,WAAW;GACnD,MAAM,gBAAsB;IAC3B,MAAM,MAAM,QAAQ,QAAQ,KAAK;IACjC,IAAI,QAAQ,IAAI,QAAQ,OAAO,KAAK,CAAC;IACrC,OAAO,QAAQ,MAAM;GACtB;GACA,MAAM,cAAoB;IACzB,QAAQ,oBAAoB,SAAS,OAAO;IAC5C;IACA,QAAQ,OAAO;GAChB;GACA,QAAQ,KAAK,KAAK;GAClB,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;EAC1D,CAAC;CACF,EACD;AACD;AAEA,MAAM,oBAA0B,CAAC;;;;;;;;AASjC,MAAa,qBACZ,YACA,QAAsB,KAAK,QACV;CACjB,IAAI,CAAC,YACJ,OAAO,EAAE,MAAM,YAAY,YAAY;CAGxC,MAAM,UAAyB,CAAC;CAChC,IAAI,WAAW,mBACd,QAAQ,KAAK,aAAa,WAAW,mBAAmB,SAAS,CAAC;CACnE,IAAI,WAAW,iBAAiB,QAAQ,KAAK,aAAa,WAAW,iBAAiB,OAAO,CAAC;CAC9F,MAAM,YAAY,WAAW,gBAAgB,gBAAgB,WAAW,aAAa,IAAI,KAAA;CAEzF,OAAO,EACN,MAAM,KAAK,QAAQ;EAClB,IAAI,QAAQ,SAAS,MAAM,OAAO;EAClC,MAAM,qBAAqB,YAAY,MAAM,UAAU,QAAQ,MAAM,IAAI;EACzE,IAAI;GACH,IAAI,QAAQ,QAAQ,MAAM,cAAc,SAAS,OAAO,MAAM;GAC9D,OAAO;EACR,SAAS,KAAK;GACb,mBAAmB;GACnB,MAAM;EACP;CACD,EACD;AACD;;AAGA,MAAa,cACZ,KACA,SACA,UACiB;CACjB,IAAI,UAAU,IAAI,IAAI,QAAQ,EAAE;CAChC,IAAI,CAAC,SAAS;EACb,UAAU,kBAAkB,QAAQ,aAAa,WAAW,KAAK;EACjE,IAAI,IAAI,QAAQ,IAAI,OAAO;CAC5B;CACA,OAAO;AACR"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { ProviderHttpError } from "../adapters/http/fetchJson.js";
|
|
2
|
+
//#region src/surfacing/retryPolicy.ts
|
|
3
|
+
/** Reason message the engine's own read-timeout abort uses; shared so classification and the abort reason cannot drift apart. */
|
|
4
|
+
const PROVIDER_READ_TIMEOUT_MESSAGE = "analytics: provider read timed out";
|
|
5
|
+
const isAbortError = (err) => err instanceof Error && (err.name === "AbortError" || err.message === "analytics: provider read timed out");
|
|
6
|
+
/** Retry classification for provider reads: 429 and 5xx back off (transient quota/server trouble), other HTTP statuses are the caller's bug and never retry, aborts never retry, anything non-HTTP is presumed a transient network failure and gets one more try. */
|
|
7
|
+
const shouldRetryProviderError = (err, attempt) => {
|
|
8
|
+
if (err instanceof ProviderHttpError) return (err.status === 429 || err.status >= 500) && attempt < 2;
|
|
9
|
+
if (isAbortError(err)) return false;
|
|
10
|
+
return attempt < 1;
|
|
11
|
+
};
|
|
12
|
+
//#endregion
|
|
13
|
+
export { PROVIDER_READ_TIMEOUT_MESSAGE, shouldRetryProviderError };
|
|
14
|
+
|
|
15
|
+
//# sourceMappingURL=retryPolicy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retryPolicy.js","names":[],"sources":["../../src/surfacing/retryPolicy.ts"],"sourcesContent":["import { ProviderHttpError } from '../adapters/http/fetchJson'\n\n/** Reason message the engine's own read-timeout abort uses; shared so classification and the abort reason cannot drift apart. */\nexport const PROVIDER_READ_TIMEOUT_MESSAGE = 'analytics: provider read timed out'\n\nconst isAbortError = (err: unknown): boolean =>\n\terr instanceof Error &&\n\t(err.name === 'AbortError' || err.message === PROVIDER_READ_TIMEOUT_MESSAGE)\n\n/** Retry classification for provider reads: 429 and 5xx back off (transient quota/server trouble), other HTTP statuses are the caller's bug and never retry, aborts never retry, anything non-HTTP is presumed a transient network failure and gets one more try. */\nexport const shouldRetryProviderError = (err: unknown, attempt: number): boolean => {\n\tif (err instanceof ProviderHttpError) {\n\t\treturn (err.status === 429 || err.status >= 500) && attempt < 2\n\t}\n\tif (isAbortError(err)) return false\n\treturn attempt < 1\n}\n"],"mappings":";;;AAGA,MAAa,gCAAgC;AAE7C,MAAM,gBAAgB,QACrB,eAAe,UACd,IAAI,SAAS,gBAAgB,IAAI,YAAA;;AAGnC,MAAa,4BAA4B,KAAc,YAA6B;CACnF,IAAI,eAAe,mBAClB,QAAQ,IAAI,WAAW,OAAO,IAAI,UAAU,QAAQ,UAAU;CAE/D,IAAI,aAAa,GAAG,GAAG,OAAO;CAC9B,OAAO,UAAU;AAClB"}
|
package/dist/sync/collection.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { providerRowAccess } from "../providers/access.js";
|
|
1
2
|
//#region src/sync/collection.ts
|
|
2
3
|
const METRIC_FIELDS = [
|
|
3
4
|
"pageviews",
|
|
@@ -14,12 +15,15 @@ const METRIC_FIELDS = [
|
|
|
14
15
|
* substrate for custom reporting rather than something to browse and edit; pass
|
|
15
16
|
* `sync: { hidden: false }` to surface it. `admin.hidden` only hides the nav entry, so
|
|
16
17
|
* the collection stays fully queryable through the REST/local API either way. The unique
|
|
17
|
-
* (source, date) index makes re-syncs idempotent and is cross-DB clean (
|
|
18
|
-
* are required, so NOT NULL on Postgres, which would otherwise treat repeated rows
|
|
19
|
-
* distinct). Metric fields are nullable so a provider that omits a metric is
|
|
20
|
-
* distinguishable from a real zero.
|
|
18
|
+
* (source, date, scope) index makes re-syncs idempotent and is cross-DB clean (all key
|
|
19
|
+
* columns are required, so NOT NULL on Postgres, which would otherwise treat repeated rows
|
|
20
|
+
* as distinct). Metric fields are nullable so a provider that omits a metric is
|
|
21
|
+
* distinguishable from a real zero. Read access reuses `providerRowAccess`: unscoped
|
|
22
|
+
* installs keep the any-authenticated-user behavior, scoped installs constrain reads to the
|
|
23
|
+
* request's resolved scope (or lift the constraint for a platform-read grant). Writes stay
|
|
24
|
+
* locked to the sync job, which writes with `overrideAccess: true`.
|
|
21
25
|
*/
|
|
22
|
-
const syncCollection = (slug, hidden
|
|
26
|
+
const syncCollection = (slug, hidden, access) => ({
|
|
23
27
|
slug,
|
|
24
28
|
admin: {
|
|
25
29
|
hidden,
|
|
@@ -32,7 +36,7 @@ const syncCollection = (slug, hidden = true) => ({
|
|
|
32
36
|
]
|
|
33
37
|
},
|
|
34
38
|
access: {
|
|
35
|
-
read: (
|
|
39
|
+
read: providerRowAccess(access),
|
|
36
40
|
create: () => false,
|
|
37
41
|
update: () => false,
|
|
38
42
|
delete: () => false
|
|
@@ -50,6 +54,14 @@ const syncCollection = (slug, hidden = true) => ({
|
|
|
50
54
|
required: true,
|
|
51
55
|
index: true
|
|
52
56
|
},
|
|
57
|
+
{
|
|
58
|
+
name: "scope",
|
|
59
|
+
type: "text",
|
|
60
|
+
required: true,
|
|
61
|
+
defaultValue: "",
|
|
62
|
+
index: true,
|
|
63
|
+
validate: ((value) => typeof value === "string" ? true : "invalid scope")
|
|
64
|
+
},
|
|
53
65
|
...METRIC_FIELDS.map((name) => ({
|
|
54
66
|
name,
|
|
55
67
|
type: "number"
|
|
@@ -60,7 +72,11 @@ const syncCollection = (slug, hidden = true) => ({
|
|
|
60
72
|
}
|
|
61
73
|
],
|
|
62
74
|
indexes: [{
|
|
63
|
-
fields: [
|
|
75
|
+
fields: [
|
|
76
|
+
"source",
|
|
77
|
+
"date",
|
|
78
|
+
"scope"
|
|
79
|
+
],
|
|
64
80
|
unique: true
|
|
65
81
|
}]
|
|
66
82
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collection.js","names":[],"sources":["../../src/sync/collection.ts"],"sourcesContent":["import type { CollectionConfig, Field } from 'payload'\n\nexport const SYNC_DEFAULT_SLUG = 'analytics-daily'\n\nexport const METRIC_FIELDS = [\n\t'pageviews',\n\t'visitors',\n\t'sessions',\n\t'avgDuration',\n\t'bounceRate',\n\t'events',\n] as const\n\nexport type SyncMetric = (typeof METRIC_FIELDS)[number]\n\n/**\n * The sync tier's queryable collection: one row per (source, date) of a provider's daily\n * metrics. It is read-only and machine-written (writes are locked to the sync job, which\n * uses overrideAccess), and hidden from the admin nav by default because it is a query\n * substrate for custom reporting rather than something to browse and edit; pass\n * `sync: { hidden: false }` to surface it. `admin.hidden` only hides the nav entry, so\n * the collection stays fully queryable through the REST/local API either way. The unique\n * (source, date) index makes re-syncs idempotent and is cross-DB clean (
|
|
1
|
+
{"version":3,"file":"collection.js","names":[],"sources":["../../src/sync/collection.ts"],"sourcesContent":["import type { CollectionConfig, Field, TextFieldSingleValidation } from 'payload'\nimport type { ProviderAccessArgs } from '../providers/access'\nimport { providerRowAccess } from '../providers/access'\n\nexport const SYNC_DEFAULT_SLUG = 'analytics-daily'\n\nexport const METRIC_FIELDS = [\n\t'pageviews',\n\t'visitors',\n\t'sessions',\n\t'avgDuration',\n\t'bounceRate',\n\t'events',\n] as const\n\nexport type SyncMetric = (typeof METRIC_FIELDS)[number]\n\n/**\n * The sync tier's queryable collection: one row per (source, date) of a provider's daily\n * metrics. It is read-only and machine-written (writes are locked to the sync job, which\n * uses overrideAccess), and hidden from the admin nav by default because it is a query\n * substrate for custom reporting rather than something to browse and edit; pass\n * `sync: { hidden: false }` to surface it. `admin.hidden` only hides the nav entry, so\n * the collection stays fully queryable through the REST/local API either way. The unique\n * (source, date, scope) index makes re-syncs idempotent and is cross-DB clean (all key\n * columns are required, so NOT NULL on Postgres, which would otherwise treat repeated rows\n * as distinct). Metric fields are nullable so a provider that omits a metric is\n * distinguishable from a real zero. Read access reuses `providerRowAccess`: unscoped\n * installs keep the any-authenticated-user behavior, scoped installs constrain reads to the\n * request's resolved scope (or lift the constraint for a platform-read grant). Writes stay\n * locked to the sync job, which writes with `overrideAccess: true`.\n */\nexport const syncCollection = (\n\tslug: string,\n\thidden: boolean,\n\taccess: ProviderAccessArgs\n): CollectionConfig => ({\n\tslug,\n\tadmin: {\n\t\thidden,\n\t\tuseAsTitle: 'source',\n\t\tdefaultColumns: ['source', 'date', 'pageviews', 'visitors'],\n\t},\n\taccess: {\n\t\tread: providerRowAccess(access),\n\t\tcreate: () => false,\n\t\tupdate: () => false,\n\t\tdelete: () => false,\n\t},\n\tfields: [\n\t\t{ name: 'source', type: 'text', required: true, index: true },\n\t\t{ name: 'date', type: 'date', required: true, index: true },\n\t\t{\n\t\t\tname: 'scope',\n\t\t\ttype: 'text',\n\t\t\trequired: true,\n\t\t\tdefaultValue: '',\n\t\t\tindex: true,\n\t\t\t// '' is the deliberate sentinel for \"no scope\"; Payload's default text\n\t\t\t// validator treats it as failing `required`, so accept any string.\n\t\t\tvalidate: ((value) =>\n\t\t\t\ttypeof value === 'string' ? true : 'invalid scope') as TextFieldSingleValidation,\n\t\t},\n\t\t...METRIC_FIELDS.map((name): Field => ({ name, type: 'number' })),\n\t\t{ name: 'syncedAt', type: 'date' },\n\t],\n\tindexes: [{ fields: ['source', 'date', 'scope'], unique: true }],\n})\n"],"mappings":";;AAMA,MAAa,gBAAgB;CAC5B;CACA;CACA;CACA;CACA;CACA;AACD;;;;;;;;;;;;;;;;AAmBA,MAAa,kBACZ,MACA,QACA,YACuB;CACvB;CACA,OAAO;EACN;EACA,YAAY;EACZ,gBAAgB;GAAC;GAAU;GAAQ;GAAa;EAAU;CAC3D;CACA,QAAQ;EACP,MAAM,kBAAkB,MAAM;EAC9B,cAAc;EACd,cAAc;EACd,cAAc;CACf;CACA,QAAQ;EACP;GAAE,MAAM;GAAU,MAAM;GAAQ,UAAU;GAAM,OAAO;EAAK;EAC5D;GAAE,MAAM;GAAQ,MAAM;GAAQ,UAAU;GAAM,OAAO;EAAK;EAC1D;GACC,MAAM;GACN,MAAM;GACN,UAAU;GACV,cAAc;GACd,OAAO;GAGP,YAAY,UACX,OAAO,UAAU,WAAW,OAAO;EACrC;EACA,GAAG,cAAc,KAAK,UAAiB;GAAE;GAAM,MAAM;EAAS,EAAE;EAChE;GAAE,MAAM;GAAY,MAAM;EAAO;CAClC;CACA,SAAS,CAAC;EAAE,QAAQ;GAAC;GAAU;GAAQ;EAAO;EAAG,QAAQ;CAAK,CAAC;AAChE"}
|
package/dist/sync/syncTask.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { getRuntime, resolveRegistryFor } from "../plugin/runtime.js";
|
|
1
2
|
import { supportsGranularity } from "../core/granularity.js";
|
|
2
|
-
import {
|
|
3
|
+
import { resolveScopeList } from "../core/scopeList.js";
|
|
3
4
|
import { METRIC_FIELDS } from "./collection.js";
|
|
4
5
|
//#region src/sync/syncTask.ts
|
|
5
6
|
const SYNC_TASK_SLUG = "analytics-sync";
|
|
@@ -9,14 +10,15 @@ const DAY_MS = 864e5;
|
|
|
9
10
|
* `date`, each present numeric metric is copied, absent metrics are omitted (stored null).
|
|
10
11
|
* Returns null when the row has no usable timestamp, so a malformed row is skipped not written.
|
|
11
12
|
*/
|
|
12
|
-
const toSyncRow = (source, row,
|
|
13
|
+
const toSyncRow = (source, row, opts) => {
|
|
13
14
|
if (!row.timestamp) return null;
|
|
14
15
|
const date = new Date(row.timestamp);
|
|
15
16
|
if (Number.isNaN(date.getTime())) return null;
|
|
16
17
|
const doc = {
|
|
17
18
|
source,
|
|
18
19
|
date,
|
|
19
|
-
syncedAt
|
|
20
|
+
syncedAt: opts.syncedAt,
|
|
21
|
+
scope: opts.scope ?? ""
|
|
20
22
|
};
|
|
21
23
|
for (const metric of METRIC_FIELDS) {
|
|
22
24
|
const value = row.metrics[metric];
|
|
@@ -24,11 +26,24 @@ const toSyncRow = (source, row, syncedAt) => {
|
|
|
24
26
|
}
|
|
25
27
|
return doc;
|
|
26
28
|
};
|
|
29
|
+
/**
|
|
30
|
+
* True when `adapterIds` is unset, or `adapterId` is listed exactly, or (for a
|
|
31
|
+
* runtime instance id shaped `<provider>:<docId>`) its provider type is listed.
|
|
32
|
+
*/
|
|
33
|
+
const matchesAdapterFilter = (adapterIds, adapterId) => {
|
|
34
|
+
if (!adapterIds) return true;
|
|
35
|
+
if (adapterIds.includes(adapterId)) return true;
|
|
36
|
+
return adapterIds.includes(adapterId.split(":")[0] ?? adapterId);
|
|
37
|
+
};
|
|
27
38
|
const startOfUtcDay = (d) => new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
|
|
28
39
|
const upsertDailyRow = async (req, collectionSlug, row) => {
|
|
29
40
|
const found = (await req.payload.find({
|
|
30
41
|
collection: collectionSlug,
|
|
31
|
-
where: { and: [
|
|
42
|
+
where: { and: [
|
|
43
|
+
{ source: { equals: row.source } },
|
|
44
|
+
{ date: { equals: row.date.toISOString() } },
|
|
45
|
+
{ scope: { equals: row.scope } }
|
|
46
|
+
] },
|
|
32
47
|
limit: 1,
|
|
33
48
|
depth: 0,
|
|
34
49
|
overrideAccess: true
|
|
@@ -50,10 +65,12 @@ const upsertDailyRow = async (req, collectionSlug, row) => {
|
|
|
50
65
|
};
|
|
51
66
|
/**
|
|
52
67
|
* Opt-in Payload task that ETLs each configured provider adapter's last `lookbackDays` of
|
|
53
|
-
* daily metrics into the sync collection (one upserted row per source + day),
|
|
54
|
-
* through the surfacing engine so the pull rides its queue / backoff / cache.
|
|
55
|
-
* excluded (its data already lives in the rollups collection); each provider
|
|
56
|
-
* own try/catch so one failure does not abort the rest.
|
|
68
|
+
* daily metrics into the sync collection (one upserted row per scope + source + day),
|
|
69
|
+
* reading through the surfacing engine so the pull rides its queue / backoff / cache.
|
|
70
|
+
* Native is excluded (its data already lives in the rollups collection); each provider
|
|
71
|
+
* runs in its own try/catch so one failure does not abort the rest. With `scopes`
|
|
72
|
+
* configured, the whole per-adapter pull runs once per tenant scope (plus the
|
|
73
|
+
* install-wide scope) so a multi-tenant install syncs every tenant, not just the null one.
|
|
57
74
|
*/
|
|
58
75
|
const syncTask = (opts) => ({
|
|
59
76
|
slug: SYNC_TASK_SLUG,
|
|
@@ -70,40 +87,58 @@ const syncTask = (opts) => ({
|
|
|
70
87
|
};
|
|
71
88
|
let synced = 0;
|
|
72
89
|
let failed = 0;
|
|
73
|
-
const
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
req,
|
|
77
|
-
scope
|
|
78
|
-
});
|
|
79
|
-
for (const adapter of registry.all()) {
|
|
80
|
-
if (adapter.id === "native") continue;
|
|
81
|
-
if (opts.adapterIds && !opts.adapterIds.includes(adapter.id)) continue;
|
|
82
|
-
if (!adapter.isConfigured() || !supportsGranularity(adapter.capabilities, "day")) continue;
|
|
83
|
-
const metrics = METRIC_FIELDS.filter((m) => adapter.capabilities.metrics.has(m));
|
|
84
|
-
if (metrics.length === 0) continue;
|
|
85
|
-
let result;
|
|
90
|
+
const scopeList = await resolveScopeList(opts.scopes, req.payload);
|
|
91
|
+
for (const scope of scopeList) {
|
|
92
|
+
let registry;
|
|
86
93
|
try {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
scope: scope ?? void 0
|
|
94
|
+
registry = await resolveRegistryFor(runtime, {
|
|
95
|
+
payload: req.payload,
|
|
96
|
+
req,
|
|
97
|
+
scope
|
|
92
98
|
});
|
|
93
99
|
} catch (err) {
|
|
94
100
|
failed++;
|
|
95
|
-
req.payload.logger.warn(`analytics sync:
|
|
101
|
+
req.payload.logger.warn(`analytics sync: registry resolution failed for scope "${scope ?? "install-wide"}": ${String(err)}`);
|
|
96
102
|
continue;
|
|
97
103
|
}
|
|
98
|
-
for (const
|
|
99
|
-
|
|
100
|
-
if (!
|
|
104
|
+
for (const adapter of registry.all()) {
|
|
105
|
+
if (adapter.id === "native") continue;
|
|
106
|
+
if (!matchesAdapterFilter(opts.adapterIds, adapter.id)) continue;
|
|
107
|
+
if (scope !== null && runtime.configAdapterIds.has(adapter.id) && !adapter.capabilities.scopedQueries) continue;
|
|
108
|
+
if (!adapter.isConfigured() || !supportsGranularity(adapter.capabilities, "day")) continue;
|
|
109
|
+
const metrics = METRIC_FIELDS.filter((m) => adapter.capabilities.metrics.has(m));
|
|
110
|
+
if (metrics.length === 0) continue;
|
|
111
|
+
let result;
|
|
101
112
|
try {
|
|
102
|
-
await
|
|
103
|
-
|
|
113
|
+
result = await runtime.engine.read(adapter, {
|
|
114
|
+
metrics,
|
|
115
|
+
dateRange,
|
|
116
|
+
granularity: "day",
|
|
117
|
+
scope: scope ?? void 0
|
|
118
|
+
});
|
|
104
119
|
} catch (err) {
|
|
105
120
|
failed++;
|
|
106
|
-
req.payload.logger.warn(`analytics sync:
|
|
121
|
+
req.payload.logger.warn(`analytics sync: adapter "${adapter.id}" read failed (scope "${scope ?? ""}"): ${String(err)}`);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (result.meta.stale) {
|
|
125
|
+
failed++;
|
|
126
|
+
req.payload.logger.warn(`analytics sync: adapter "${adapter.id}" read was stale-served, skipping sync (scope "${scope ?? ""}")`);
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
for (const row of result.rows) {
|
|
130
|
+
const doc = toSyncRow(adapter.id, row, {
|
|
131
|
+
syncedAt: now,
|
|
132
|
+
scope: scope ?? ""
|
|
133
|
+
});
|
|
134
|
+
if (!doc) continue;
|
|
135
|
+
try {
|
|
136
|
+
await upsertDailyRow(req, opts.collectionSlug, doc);
|
|
137
|
+
synced++;
|
|
138
|
+
} catch (err) {
|
|
139
|
+
failed++;
|
|
140
|
+
req.payload.logger.warn(`analytics sync: upsert for "${adapter.id}" on ${doc.date.toISOString()} failed: ${String(err)}`);
|
|
141
|
+
}
|
|
107
142
|
}
|
|
108
143
|
}
|
|
109
144
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"syncTask.js","names":[],"sources":["../../src/sync/syncTask.ts"],"sourcesContent":["import type { PayloadRequest, TaskConfig } from 'payload'\nimport type { AnalyticsResult, AnalyticsRow } from '../core/contract'\nimport { supportsGranularity } from '../core/granularity'\nimport { getRuntime, resolveRegistryFor, resolveScopeFor } from '../plugin/runtime'\nimport { METRIC_FIELDS, type SyncMetric } from './collection'\n\nexport const SYNC_TASK_SLUG = 'analytics-sync'\n\nconst DAY_MS = 86_400_000\n\nexport type SyncRow = { source: string; date: Date; syncedAt: Date } & Partial<\n\tRecord<SyncMetric, number>\n>\n\nexport interface SyncTaskOptions {\n\tcron: string\n\tlookbackDays: number\n\tcollectionSlug: string\n\tadapterIds?: string[]\n}\n\n/**\n * Map one daily `AnalyticsRow` to a sync-collection doc: the row's UTC-day timestamp becomes\n * `date`, each present numeric metric is copied, absent metrics are omitted (stored null).\n * Returns null when the row has no usable timestamp, so a malformed row is skipped not written.\n */\nexport const toSyncRow = (source: string, row: AnalyticsRow, syncedAt: Date): SyncRow | null => {\n\tif (!row.timestamp) {\n\t\treturn null\n\t}\n\tconst date = new Date(row.timestamp)\n\tif (Number.isNaN(date.getTime())) {\n\t\treturn null\n\t}\n\tconst doc: SyncRow = { source, date, syncedAt }\n\tfor (const metric of METRIC_FIELDS) {\n\t\tconst value = row.metrics[metric]\n\t\tif (typeof value === 'number') {\n\t\t\tdoc[metric] = value\n\t\t}\n\t}\n\treturn doc\n}\n\nconst startOfUtcDay = (d: Date): Date =>\n\tnew Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()))\n\nconst upsertDailyRow = async (\n\treq: PayloadRequest,\n\tcollectionSlug: string,\n\trow: SyncRow\n): Promise<void> => {\n\tconst existing = await req.payload.find({\n\t\tcollection: collectionSlug as never,\n\t\twhere: {\n\t\t\tand: [{ source: { equals: row.source } }, { date: { equals: row.date.toISOString() } }],\n\t\t},\n\t\tlimit: 1,\n\t\tdepth: 0,\n\t\toverrideAccess: true,\n\t})\n\tconst found = existing.docs[0] as { id: string | number } | undefined\n\tif (found) {\n\t\tawait req.payload.update({\n\t\t\tcollection: collectionSlug as never,\n\t\t\tid: found.id,\n\t\t\tdata: row as never,\n\t\t\toverrideAccess: true,\n\t\t})\n\t\treturn\n\t}\n\tawait req.payload.create({\n\t\tcollection: collectionSlug as never,\n\t\tdata: row as never,\n\t\toverrideAccess: true,\n\t})\n}\n\n/**\n * Opt-in Payload task that ETLs each configured provider adapter's last `lookbackDays` of\n * daily metrics into the sync collection (one upserted row per source + day), reading\n * through the surfacing engine so the pull rides its queue / backoff / cache. Native is\n * excluded (its data already lives in the rollups collection); each provider runs in its\n * own try/catch so one failure does not abort the rest.\n */\nexport const syncTask = (\n\topts: SyncTaskOptions\n): TaskConfig<{ input: Record<string, never>; output: { synced: number; failed: number } }> => ({\n\tslug: SYNC_TASK_SLUG,\n\thandler: async ({ req }) => {\n\t\tconst runtime = getRuntime(req.payload)\n\t\tif (!runtime) {\n\t\t\treturn { output: { synced: 0, failed: 0 } }\n\t\t}\n\t\tconst now = new Date()\n\t\tconst dateRange = {\n\t\t\tstart: new Date(startOfUtcDay(now).getTime() - (opts.lookbackDays - 1) * DAY_MS),\n\t\t\tend: now,\n\t\t}\n\t\tlet synced = 0\n\t\tlet failed = 0\n\t\t// A job req may lack the request shape an app's scopeResolver expects;\n\t\t// degrade to the install scope rather than failing the whole run.\n\t\tconst scope = await resolveScopeFor(runtime, req).catch(() => null)\n\t\tconst registry = await resolveRegistryFor(runtime, { payload: req.payload, req, scope })\n\t\tfor (const adapter of registry.all()) {\n\t\t\tif (adapter.id === 'native') {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif (opts.adapterIds && !opts.adapterIds.includes(adapter.id)) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif (!adapter.isConfigured() || !supportsGranularity(adapter.capabilities, 'day')) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tconst metrics = METRIC_FIELDS.filter((m) => adapter.capabilities.metrics.has(m))\n\t\t\tif (metrics.length === 0) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tlet result: AnalyticsResult\n\t\t\ttry {\n\t\t\t\tresult = await runtime.engine.read(adapter, {\n\t\t\t\t\tmetrics,\n\t\t\t\t\tdateRange,\n\t\t\t\t\tgranularity: 'day',\n\t\t\t\t\tscope: scope ?? undefined,\n\t\t\t\t})\n\t\t\t} catch (err) {\n\t\t\t\tfailed++\n\t\t\t\treq.payload.logger.warn(\n\t\t\t\t\t`analytics sync: adapter \"${adapter.id}\" read failed: ${String(err)}`\n\t\t\t\t)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor (const row of result.rows) {\n\t\t\t\tconst doc = toSyncRow(adapter.id, row, now)\n\t\t\t\tif (!doc) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\ttry {\n\t\t\t\t\tawait upsertDailyRow(req, opts.collectionSlug, doc)\n\t\t\t\t\tsynced++\n\t\t\t\t} catch (err) {\n\t\t\t\t\tfailed++\n\t\t\t\t\treq.payload.logger.warn(\n\t\t\t\t\t\t`analytics sync: upsert for \"${adapter.id}\" on ${doc.date.toISOString()} failed: ${String(err)}`\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn { output: { synced, failed } }\n\t},\n\tschedule: [{ cron: opts.cron, queue: 'default' }],\n})\n"],"mappings":";;;;AAMA,MAAa,iBAAiB;AAE9B,MAAM,SAAS;;;;;;AAkBf,MAAa,aAAa,QAAgB,KAAmB,aAAmC;CAC/F,IAAI,CAAC,IAAI,WACR,OAAO;CAER,MAAM,OAAO,IAAI,KAAK,IAAI,SAAS;CACnC,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAC9B,OAAO;CAER,MAAM,MAAe;EAAE;EAAQ;EAAM;CAAS;CAC9C,KAAK,MAAM,UAAU,eAAe;EACnC,MAAM,QAAQ,IAAI,QAAQ;EAC1B,IAAI,OAAO,UAAU,UACpB,IAAI,UAAU;CAEhB;CACA,OAAO;AACR;AAEA,MAAM,iBAAiB,MACtB,IAAI,KAAK,KAAK,IAAI,EAAE,eAAe,GAAG,EAAE,YAAY,GAAG,EAAE,WAAW,CAAC,CAAC;AAEvE,MAAM,iBAAiB,OACtB,KACA,gBACA,QACmB;CAUnB,MAAM,SAAQ,MATS,IAAI,QAAQ,KAAK;EACvC,YAAY;EACZ,OAAO,EACN,KAAK,CAAC,EAAE,QAAQ,EAAE,QAAQ,IAAI,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,IAAI,KAAK,YAAY,EAAE,EAAE,CAAC,EACvF;EACA,OAAO;EACP,OAAO;EACP,gBAAgB;CACjB,CAAC,GACsB,KAAK;CAC5B,IAAI,OAAO;EACV,MAAM,IAAI,QAAQ,OAAO;GACxB,YAAY;GACZ,IAAI,MAAM;GACV,MAAM;GACN,gBAAgB;EACjB,CAAC;EACD;CACD;CACA,MAAM,IAAI,QAAQ,OAAO;EACxB,YAAY;EACZ,MAAM;EACN,gBAAgB;CACjB,CAAC;AACF;;;;;;;;AASA,MAAa,YACZ,UAC+F;CAC/F,MAAM;CACN,SAAS,OAAO,EAAE,UAAU;EAC3B,MAAM,UAAU,WAAW,IAAI,OAAO;EACtC,IAAI,CAAC,SACJ,OAAO,EAAE,QAAQ;GAAE,QAAQ;GAAG,QAAQ;EAAE,EAAE;EAE3C,MAAM,sBAAM,IAAI,KAAK;EACrB,MAAM,YAAY;GACjB,uBAAO,IAAI,KAAK,cAAc,GAAG,EAAE,QAAQ,KAAK,KAAK,eAAe,KAAK,MAAM;GAC/E,KAAK;EACN;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EAGb,MAAM,QAAQ,MAAM,gBAAgB,SAAS,GAAG,EAAE,YAAY,IAAI;EAClE,MAAM,WAAW,MAAM,mBAAmB,SAAS;GAAE,SAAS,IAAI;GAAS;GAAK;EAAM,CAAC;EACvF,KAAK,MAAM,WAAW,SAAS,IAAI,GAAG;GACrC,IAAI,QAAQ,OAAO,UAClB;GAED,IAAI,KAAK,cAAc,CAAC,KAAK,WAAW,SAAS,QAAQ,EAAE,GAC1D;GAED,IAAI,CAAC,QAAQ,aAAa,KAAK,CAAC,oBAAoB,QAAQ,cAAc,KAAK,GAC9E;GAED,MAAM,UAAU,cAAc,QAAQ,MAAM,QAAQ,aAAa,QAAQ,IAAI,CAAC,CAAC;GAC/E,IAAI,QAAQ,WAAW,GACtB;GAED,IAAI;GACJ,IAAI;IACH,SAAS,MAAM,QAAQ,OAAO,KAAK,SAAS;KAC3C;KACA;KACA,aAAa;KACb,OAAO,SAAS,KAAA;IACjB,CAAC;GACF,SAAS,KAAK;IACb;IACA,IAAI,QAAQ,OAAO,KAClB,4BAA4B,QAAQ,GAAG,iBAAiB,OAAO,GAAG,GACnE;IACA;GACD;GACA,KAAK,MAAM,OAAO,OAAO,MAAM;IAC9B,MAAM,MAAM,UAAU,QAAQ,IAAI,KAAK,GAAG;IAC1C,IAAI,CAAC,KACJ;IAED,IAAI;KACH,MAAM,eAAe,KAAK,KAAK,gBAAgB,GAAG;KAClD;IACD,SAAS,KAAK;KACb;KACA,IAAI,QAAQ,OAAO,KAClB,+BAA+B,QAAQ,GAAG,OAAO,IAAI,KAAK,YAAY,EAAE,WAAW,OAAO,GAAG,GAC9F;IACD;GACD;EACD;EACA,OAAO,EAAE,QAAQ;GAAE;GAAQ;EAAO,EAAE;CACrC;CACA,UAAU,CAAC;EAAE,MAAM,KAAK;EAAM,OAAO;CAAU,CAAC;AACjD"}
|
|
1
|
+
{"version":3,"file":"syncTask.js","names":[],"sources":["../../src/sync/syncTask.ts"],"sourcesContent":["import type { PayloadRequest, TaskConfig } from 'payload'\nimport type { AnalyticsResult, AnalyticsRow } from '../core/contract'\nimport { supportsGranularity } from '../core/granularity'\nimport type { ScopesResolver } from '../core/options'\nimport { resolveScopeList } from '../core/scopeList'\nimport { getRuntime, resolveRegistryFor } from '../plugin/runtime'\nimport { METRIC_FIELDS, type SyncMetric } from './collection'\n\nexport const SYNC_TASK_SLUG = 'analytics-sync'\n\nconst DAY_MS = 86_400_000\n\nexport type SyncRow = { source: string; date: Date; syncedAt: Date; scope: string } & Partial<\n\tRecord<SyncMetric, number>\n>\n\nexport interface SyncTaskOptions {\n\tcron: string\n\tlookbackDays: number\n\tcollectionSlug: string\n\t/** Entries match a config adapter id exactly, and a runtime provider instance (`<provider>:<docId>`) by its provider type. */\n\tadapterIds?: string[]\n\t/** Tenant scopes to fan out over, in addition to the install-wide scope every run already covers. */\n\tscopes?: ScopesResolver\n}\n\n/**\n * Map one daily `AnalyticsRow` to a sync-collection doc: the row's UTC-day timestamp becomes\n * `date`, each present numeric metric is copied, absent metrics are omitted (stored null).\n * Returns null when the row has no usable timestamp, so a malformed row is skipped not written.\n */\nexport const toSyncRow = (\n\tsource: string,\n\trow: AnalyticsRow,\n\topts: { syncedAt: Date; scope?: string }\n): SyncRow | null => {\n\tif (!row.timestamp) {\n\t\treturn null\n\t}\n\tconst date = new Date(row.timestamp)\n\tif (Number.isNaN(date.getTime())) {\n\t\treturn null\n\t}\n\tconst doc: SyncRow = { source, date, syncedAt: opts.syncedAt, scope: opts.scope ?? '' }\n\tfor (const metric of METRIC_FIELDS) {\n\t\tconst value = row.metrics[metric]\n\t\tif (typeof value === 'number') {\n\t\t\tdoc[metric] = value\n\t\t}\n\t}\n\treturn doc\n}\n\n/**\n * True when `adapterIds` is unset, or `adapterId` is listed exactly, or (for a\n * runtime instance id shaped `<provider>:<docId>`) its provider type is listed.\n */\nexport const matchesAdapterFilter = (\n\tadapterIds: string[] | undefined,\n\tadapterId: string\n): boolean => {\n\tif (!adapterIds) {\n\t\treturn true\n\t}\n\tif (adapterIds.includes(adapterId)) {\n\t\treturn true\n\t}\n\treturn adapterIds.includes(adapterId.split(':')[0] ?? adapterId)\n}\n\nconst startOfUtcDay = (d: Date): Date =>\n\tnew Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()))\n\nconst upsertDailyRow = async (\n\treq: PayloadRequest,\n\tcollectionSlug: string,\n\trow: SyncRow\n): Promise<void> => {\n\tconst existing = await req.payload.find({\n\t\tcollection: collectionSlug as never,\n\t\twhere: {\n\t\t\tand: [\n\t\t\t\t{ source: { equals: row.source } },\n\t\t\t\t{ date: { equals: row.date.toISOString() } },\n\t\t\t\t{ scope: { equals: row.scope } },\n\t\t\t],\n\t\t},\n\t\tlimit: 1,\n\t\tdepth: 0,\n\t\toverrideAccess: true,\n\t})\n\tconst found = existing.docs[0] as { id: string | number } | undefined\n\tif (found) {\n\t\tawait req.payload.update({\n\t\t\tcollection: collectionSlug as never,\n\t\t\tid: found.id,\n\t\t\tdata: row as never,\n\t\t\toverrideAccess: true,\n\t\t})\n\t\treturn\n\t}\n\tawait req.payload.create({\n\t\tcollection: collectionSlug as never,\n\t\tdata: row as never,\n\t\toverrideAccess: true,\n\t})\n}\n\n/**\n * Opt-in Payload task that ETLs each configured provider adapter's last `lookbackDays` of\n * daily metrics into the sync collection (one upserted row per scope + source + day),\n * reading through the surfacing engine so the pull rides its queue / backoff / cache.\n * Native is excluded (its data already lives in the rollups collection); each provider\n * runs in its own try/catch so one failure does not abort the rest. With `scopes`\n * configured, the whole per-adapter pull runs once per tenant scope (plus the\n * install-wide scope) so a multi-tenant install syncs every tenant, not just the null one.\n */\nexport const syncTask = (\n\topts: SyncTaskOptions\n): TaskConfig<{ input: Record<string, never>; output: { synced: number; failed: number } }> => ({\n\tslug: SYNC_TASK_SLUG,\n\thandler: async ({ req }) => {\n\t\tconst runtime = getRuntime(req.payload)\n\t\tif (!runtime) {\n\t\t\treturn { output: { synced: 0, failed: 0 } }\n\t\t}\n\t\tconst now = new Date()\n\t\tconst dateRange = {\n\t\t\tstart: new Date(startOfUtcDay(now).getTime() - (opts.lookbackDays - 1) * DAY_MS),\n\t\t\tend: now,\n\t\t}\n\t\tlet synced = 0\n\t\tlet failed = 0\n\t\tconst scopeList = await resolveScopeList(opts.scopes, req.payload)\n\t\tfor (const scope of scopeList) {\n\t\t\tlet registry: Awaited<ReturnType<typeof resolveRegistryFor>>\n\t\t\ttry {\n\t\t\t\tregistry = await resolveRegistryFor(runtime, { payload: req.payload, req, scope })\n\t\t\t} catch (err) {\n\t\t\t\tfailed++\n\t\t\t\treq.payload.logger.warn(\n\t\t\t\t\t`analytics sync: registry resolution failed for scope \"${scope ?? 'install-wide'}\": ${String(err)}`\n\t\t\t\t)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor (const adapter of registry.all()) {\n\t\t\t\tif (adapter.id === 'native') {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif (!matchesAdapterFilter(opts.adapterIds, adapter.id)) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// A shared config adapter that cannot narrow its query to one tenant would\n\t\t\t\t// otherwise return install-wide totals here, which then get stamped as this\n\t\t\t\t// scope's row; skip it for every tenant pass, same as the read path's gate.\n\t\t\t\tif (\n\t\t\t\t\tscope !== null &&\n\t\t\t\t\truntime.configAdapterIds.has(adapter.id) &&\n\t\t\t\t\t!adapter.capabilities.scopedQueries\n\t\t\t\t) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif (!adapter.isConfigured() || !supportsGranularity(adapter.capabilities, 'day')) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tconst metrics = METRIC_FIELDS.filter((m) => adapter.capabilities.metrics.has(m))\n\t\t\t\tif (metrics.length === 0) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tlet result: AnalyticsResult\n\t\t\t\ttry {\n\t\t\t\t\tresult = await runtime.engine.read(adapter, {\n\t\t\t\t\t\tmetrics,\n\t\t\t\t\t\tdateRange,\n\t\t\t\t\t\tgranularity: 'day',\n\t\t\t\t\t\tscope: scope ?? undefined,\n\t\t\t\t\t})\n\t\t\t\t} catch (err) {\n\t\t\t\t\tfailed++\n\t\t\t\t\treq.payload.logger.warn(\n\t\t\t\t\t\t`analytics sync: adapter \"${adapter.id}\" read failed (scope \"${scope ?? ''}\"): ${String(err)}`\n\t\t\t\t\t)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif (result.meta.stale) {\n\t\t\t\t\tfailed++\n\t\t\t\t\treq.payload.logger.warn(\n\t\t\t\t\t\t`analytics sync: adapter \"${adapter.id}\" read was stale-served, skipping sync (scope \"${scope ?? ''}\")`\n\t\t\t\t\t)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfor (const row of result.rows) {\n\t\t\t\t\tconst doc = toSyncRow(adapter.id, row, { syncedAt: now, scope: scope ?? '' })\n\t\t\t\t\tif (!doc) {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\ttry {\n\t\t\t\t\t\tawait upsertDailyRow(req, opts.collectionSlug, doc)\n\t\t\t\t\t\tsynced++\n\t\t\t\t\t} catch (err) {\n\t\t\t\t\t\tfailed++\n\t\t\t\t\t\treq.payload.logger.warn(\n\t\t\t\t\t\t\t`analytics sync: upsert for \"${adapter.id}\" on ${doc.date.toISOString()} failed: ${String(err)}`\n\t\t\t\t\t\t)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn { output: { synced, failed } }\n\t},\n\tschedule: [{ cron: opts.cron, queue: 'default' }],\n})\n"],"mappings":";;;;;AAQA,MAAa,iBAAiB;AAE9B,MAAM,SAAS;;;;;;AAqBf,MAAa,aACZ,QACA,KACA,SACoB;CACpB,IAAI,CAAC,IAAI,WACR,OAAO;CAER,MAAM,OAAO,IAAI,KAAK,IAAI,SAAS;CACnC,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAC9B,OAAO;CAER,MAAM,MAAe;EAAE;EAAQ;EAAM,UAAU,KAAK;EAAU,OAAO,KAAK,SAAS;CAAG;CACtF,KAAK,MAAM,UAAU,eAAe;EACnC,MAAM,QAAQ,IAAI,QAAQ;EAC1B,IAAI,OAAO,UAAU,UACpB,IAAI,UAAU;CAEhB;CACA,OAAO;AACR;;;;;AAMA,MAAa,wBACZ,YACA,cACa;CACb,IAAI,CAAC,YACJ,OAAO;CAER,IAAI,WAAW,SAAS,SAAS,GAChC,OAAO;CAER,OAAO,WAAW,SAAS,UAAU,MAAM,GAAG,EAAE,MAAM,SAAS;AAChE;AAEA,MAAM,iBAAiB,MACtB,IAAI,KAAK,KAAK,IAAI,EAAE,eAAe,GAAG,EAAE,YAAY,GAAG,EAAE,WAAW,CAAC,CAAC;AAEvE,MAAM,iBAAiB,OACtB,KACA,gBACA,QACmB;CAcnB,MAAM,SAAQ,MAbS,IAAI,QAAQ,KAAK;EACvC,YAAY;EACZ,OAAO,EACN,KAAK;GACJ,EAAE,QAAQ,EAAE,QAAQ,IAAI,OAAO,EAAE;GACjC,EAAE,MAAM,EAAE,QAAQ,IAAI,KAAK,YAAY,EAAE,EAAE;GAC3C,EAAE,OAAO,EAAE,QAAQ,IAAI,MAAM,EAAE;EAChC,EACD;EACA,OAAO;EACP,OAAO;EACP,gBAAgB;CACjB,CAAC,GACsB,KAAK;CAC5B,IAAI,OAAO;EACV,MAAM,IAAI,QAAQ,OAAO;GACxB,YAAY;GACZ,IAAI,MAAM;GACV,MAAM;GACN,gBAAgB;EACjB,CAAC;EACD;CACD;CACA,MAAM,IAAI,QAAQ,OAAO;EACxB,YAAY;EACZ,MAAM;EACN,gBAAgB;CACjB,CAAC;AACF;;;;;;;;;;AAWA,MAAa,YACZ,UAC+F;CAC/F,MAAM;CACN,SAAS,OAAO,EAAE,UAAU;EAC3B,MAAM,UAAU,WAAW,IAAI,OAAO;EACtC,IAAI,CAAC,SACJ,OAAO,EAAE,QAAQ;GAAE,QAAQ;GAAG,QAAQ;EAAE,EAAE;EAE3C,MAAM,sBAAM,IAAI,KAAK;EACrB,MAAM,YAAY;GACjB,uBAAO,IAAI,KAAK,cAAc,GAAG,EAAE,QAAQ,KAAK,KAAK,eAAe,KAAK,MAAM;GAC/E,KAAK;EACN;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,MAAM,YAAY,MAAM,iBAAiB,KAAK,QAAQ,IAAI,OAAO;EACjE,KAAK,MAAM,SAAS,WAAW;GAC9B,IAAI;GACJ,IAAI;IACH,WAAW,MAAM,mBAAmB,SAAS;KAAE,SAAS,IAAI;KAAS;KAAK;IAAM,CAAC;GAClF,SAAS,KAAK;IACb;IACA,IAAI,QAAQ,OAAO,KAClB,yDAAyD,SAAS,eAAe,KAAK,OAAO,GAAG,GACjG;IACA;GACD;GACA,KAAK,MAAM,WAAW,SAAS,IAAI,GAAG;IACrC,IAAI,QAAQ,OAAO,UAClB;IAED,IAAI,CAAC,qBAAqB,KAAK,YAAY,QAAQ,EAAE,GACpD;IAKD,IACC,UAAU,QACV,QAAQ,iBAAiB,IAAI,QAAQ,EAAE,KACvC,CAAC,QAAQ,aAAa,eAEtB;IAED,IAAI,CAAC,QAAQ,aAAa,KAAK,CAAC,oBAAoB,QAAQ,cAAc,KAAK,GAC9E;IAED,MAAM,UAAU,cAAc,QAAQ,MAAM,QAAQ,aAAa,QAAQ,IAAI,CAAC,CAAC;IAC/E,IAAI,QAAQ,WAAW,GACtB;IAED,IAAI;IACJ,IAAI;KACH,SAAS,MAAM,QAAQ,OAAO,KAAK,SAAS;MAC3C;MACA;MACA,aAAa;MACb,OAAO,SAAS,KAAA;KACjB,CAAC;IACF,SAAS,KAAK;KACb;KACA,IAAI,QAAQ,OAAO,KAClB,4BAA4B,QAAQ,GAAG,wBAAwB,SAAS,GAAG,MAAM,OAAO,GAAG,GAC5F;KACA;IACD;IACA,IAAI,OAAO,KAAK,OAAO;KACtB;KACA,IAAI,QAAQ,OAAO,KAClB,4BAA4B,QAAQ,GAAG,iDAAiD,SAAS,GAAG,GACrG;KACA;IACD;IACA,KAAK,MAAM,OAAO,OAAO,MAAM;KAC9B,MAAM,MAAM,UAAU,QAAQ,IAAI,KAAK;MAAE,UAAU;MAAK,OAAO,SAAS;KAAG,CAAC;KAC5E,IAAI,CAAC,KACJ;KAED,IAAI;MACH,MAAM,eAAe,KAAK,KAAK,gBAAgB,GAAG;MAClD;KACD,SAAS,KAAK;MACb;MACA,IAAI,QAAQ,OAAO,KAClB,+BAA+B,QAAQ,GAAG,OAAO,IAAI,KAAK,YAAY,EAAE,WAAW,OAAO,GAAG,GAC9F;KACD;IACD;GACD;EACD;EACA,OAAO,EAAE,QAAQ;GAAE;GAAQ;EAAO,EAAE;CACrC;CACA,UAAU,CAAC;EAAE,MAAM,KAAK;EAAM,OAAO;CAAU,CAAC;AACjD"}
|
|
@@ -9,6 +9,8 @@ interface MemoryEvent {
|
|
|
9
9
|
interface MemoryAnalyticsAdapter extends AnalyticsAdapter {
|
|
10
10
|
record(event: MemoryEvent): void;
|
|
11
11
|
reset(): void;
|
|
12
|
+
/** Test seam for engine resilience: the next query() call rejects with err, then resets. */
|
|
13
|
+
failNext(err?: Error): void;
|
|
12
14
|
}
|
|
13
15
|
declare function memoryAdapter(): MemoryAnalyticsAdapter;
|
|
14
16
|
//#endregion
|