@rawdash/connector-google-play-console 0.24.0 → 0.26.0

package/dist/index.js

@@ -11,7 +11,8 @@ import {
   defineConfigFields,
   defineConnectorDoc,
   defineResources,
-  schemasFromResources
+  schemasFromResources,
+  selectActivePhases
 } from "@rawdash/core";
 import { z } from "zod";
 var configFields = defineConfigFields(
@@ -32,6 +33,11 @@ var configFields = defineConfigFields(
       label: "Lookback days (full sync)",
       description: "How many calendar days to fetch on a full sync. Defaults to 30. The Play Developer Reporting API exposes daily metrics with a typical 2-3 day reporting lag.",
       placeholder: "30"
+    }),
+    reviewLimit: z.number().int().positive().max(2e3).optional().meta({
+      label: "Review sample size",
+      description: "How many of the most-recent user reviews to emit as gplay_app_ratings samples. Defaults to 200. Reviews are fetched then ranked newest-first before this cap is applied. The Android Publisher reviews API only surfaces reviews from roughly the past week, so this is a rolling sample, not a full history.",
+      placeholder: "200"
     })
   })
 );
@@ -39,7 +45,7 @@ var doc = defineConnectorDoc({
   displayName: "Google Play Console",
   category: "engineering",
   brandColor: "#34A853",
-  tagline: "Sync daily Android app vitals from the Play Developer Reporting API - crash rate, ANR rate, ratings, and error counts.",
+  tagline: "Sync daily Android app vitals from the Play Developer Reporting API (crash rate, ANR rate, error counts) plus user review ratings from the Android Publisher API.",
   vendor: {
     name: "Google Play Console",
     domain: "play.google.com",
@@ -50,7 +56,7 @@ var doc = defineConnectorDoc({
     summary: "Authenticate against the Play Developer Reporting API and the Android Publisher API with a Google service account JSON key. The service account must be linked to your Play Console developer account.",
     setup: [
       "In Google Cloud, create a service account at IAM & Admin -> Service Accounts and download a JSON key.",
-      'Enable both the "Google Play Android Developer API" and the "Google Play Developer Reporting API" on the Cloud project.',
+      'Enable both the "Google Play Developer Reporting API" and the "Google Play Android Developer API" on the Cloud project.',
       'In Google Play Console open Setup -> API access, link the same Cloud project, then invite the service account email and grant it at least the "View app information and download bulk reports" permission for the app you want to sync.',
       'Store the service account JSON as a secret and reference it as serviceAccountJson: secret("GPLAY_SA_JSON").',
       "Set packageName to the reverse-DNS application id of the app (e.g. com.example.app)."
@@ -58,7 +64,9 @@ var doc = defineConnectorDoc({
   },
   rateLimit: "The Play Developer Reporting API enforces a per-project quota (default 60 requests per minute); 429 responses are retried with exponential backoff.",
   limitations: [
-    "Daily vitals (crash rate, ANR rate, ratings, error counts) have a 2-3 day reporting lag on the Play Developer Reporting API; incremental syncs refetch the trailing 3 days.",
+    "Daily vitals (crash rate, ANR rate, error counts) have a 2-3 day reporting lag on the Play Developer Reporting API; incremental syncs refetch the trailing 3 days. Metric days are reported on the America/Los_Angeles calendar, the only timezone the API supports for daily aggregation.",
+    "gplay_app_ratings is a rolling sample of recent reviews from the Android Publisher reviews API (default 200, configurable via reviewLimit). Each sample carries one review with its star rating (1-5) as the value; this is not the lifetime average shown on the Play Store, and the reviews API only surfaces reviews from roughly the past week.",
+    "The apps entity carries only the configured package name; the Play Store listing title is available solely through an Android Publisher edit, which this connector does not create.",
     "Install counts and earnings are not exposed through the Reporting API - Google delivers them only as monthly CSV reports in a private Cloud Storage bucket. Those metrics are out of scope for this connector and will land in a follow-up."
   ]
 });
@@ -72,8 +80,8 @@ var PHASE_ORDER = [
   "apps",
   "crash_rate",
   "anr_rate",
-  "ratings",
-  "errors"
+  "errors",
+  "reviews"
 ];
 var GPLAY_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
 function isGplayDateString(value) {
@@ -104,33 +112,42 @@ var METRIC_PHASE_CONFIGS = {
     metricSet: "crashRateMetricSet",
     metrics: ["crashRate", "distinctUsers"],
     metricName: "gplay_crash_rate_by_day",
-    primaryMetric: "crashRate"
+    primaryMetric: "crashRate",
+    responseTag: "crash_rate"
   },
   anr_rate: {
     metricSet: "anrRateMetricSet",
     metrics: ["anrRate", "distinctUsers"],
     metricName: "gplay_anr_rate_by_day",
-    primaryMetric: "anrRate"
-  },
-  ratings: {
-    metricSet: "ratingsMetricSet",
-    metrics: ["averageRating", "ratingsCount"],
-    metricName: "gplay_ratings_by_day",
-    primaryMetric: "averageRating"
+    primaryMetric: "anrRate",
+    responseTag: "anr_rate"
   },
   errors: {
     metricSet: "errorCountMetricSet",
     metrics: ["errorReportCount", "distinctUsers"],
     metricName: "gplay_error_count_by_day",
-    primaryMetric: "errorReportCount"
+    primaryMetric: "errorReportCount",
+    responseTag: "errors"
   }
 };
+var GPLAY_APP_RATINGS_METRIC = "gplay_app_ratings";
+var RESOURCE_TO_PHASE = {
+  apps: "apps",
+  gplay_crash_rate_by_day: "crash_rate",
+  gplay_anr_rate_by_day: "anr_rate",
+  gplay_error_count_by_day: "errors",
+  [GPLAY_APP_RATINGS_METRIC]: "reviews"
+};
 var SCOPES = [
   "https://www.googleapis.com/auth/playdeveloperreporting",
   "https://www.googleapis.com/auth/androidpublisher"
 ].join(" ");
 var REPORTING_BASE = "https://playdeveloperreporting.googleapis.com";
 var PUBLISHER_BASE = "https://androidpublisher.googleapis.com";
+var DAILY_TIME_ZONE = "America/Los_Angeles";
+var DEFAULT_REVIEW_LIMIT = 200;
+var REVIEWS_PAGE_SIZE = 100;
+var MAX_REVIEW_PAGES = 50;
 function base64urlFromBytes(bytes) {
   let binary = "";
   for (let i = 0; i < bytes.length; i++) {
@@ -197,11 +214,14 @@ async function buildServiceAccountJwt(serviceAccountJson) {
     body
   };
 }
+var gplayDateFormatter = new Intl.DateTimeFormat("en-CA", {
+  timeZone: DAILY_TIME_ZONE,
+  year: "numeric",
+  month: "2-digit",
+  day: "2-digit"
+});
 function toGplayDate(date) {
-  const y = date.getUTCFullYear();
-  const m = String(date.getUTCMonth() + 1).padStart(2, "0");
-  const d = String(date.getUTCDate()).padStart(2, "0");
-  return `${y}-${m}-${d}`;
+  return gplayDateFormatter.format(date);
 }
 function gplayDateToMs(gplayDate) {
   const y = gplayDate.slice(0, 4);
@@ -269,6 +289,53 @@ function rowToMetricSample(row, metricsToCollect, metricName, primaryMetric, pac
     attributes
   };
 }
+function timestampToMs(ts) {
+  if (!ts || typeof ts.seconds !== "string") {
+    return null;
+  }
+  const seconds = Number(ts.seconds);
+  if (!Number.isFinite(seconds)) {
+    return null;
+  }
+  const nanos = typeof ts.nanos === "number" ? ts.nanos : 0;
+  return Math.round(seconds * 1e3 + nanos / 1e6);
+}
+function reviewToRatingSample(review, packageName) {
+  const userComment = review.comments?.find((c) => c.userComment)?.userComment;
+  if (!userComment) {
+    return null;
+  }
+  const rating = userComment.starRating;
+  if (typeof rating !== "number" || rating < 1 || rating > 5) {
+    return null;
+  }
+  const ts = timestampToMs(userComment.lastModified);
+  if (ts === null) {
+    return null;
+  }
+  const attributes = {
+    package_name: packageName,
+    review_id: review.reviewId ?? ""
+  };
+  if (userComment.reviewerLanguage) {
+    attributes["reviewer_language"] = userComment.reviewerLanguage;
+  }
+  if (userComment.device) {
+    attributes["device"] = userComment.device;
+  }
+  if (userComment.appVersionName) {
+    attributes["app_version_name"] = userComment.appVersionName;
+  }
+  if (typeof userComment.androidOsVersion === "number") {
+    attributes["android_os_version"] = userComment.androidOsVersion;
+  }
+  return {
+    name: GPLAY_APP_RATINGS_METRIC,
+    ts,
+    value: rating,
+    attributes
+  };
+}
 var dateOnlyTimeline = z.object({
   startTime: z.object({
     year: z.number().int(),
@@ -292,38 +359,43 @@ function metricSetSchema() {
     nextPageToken: z.string().optional()
   });
 }
-var publisherListingSchema = z.object({
-  defaultLanguage: z.string().optional(),
-  listings: z.array(
+var reviewsResponseSchema = z.object({
+  reviews: z.array(
     z.object({
-      language: z.string(),
-      title: z.string().optional(),
-      shortDescription: z.string().optional(),
-      fullDescription: z.string().optional()
+      reviewId: z.string().optional(),
+      authorName: z.string().optional(),
+      comments: z.array(
+        z.object({
+          userComment: z.object({
+            text: z.string().optional(),
+            lastModified: z.object({
+              seconds: z.string().optional(),
+              nanos: z.number().optional()
+            }).optional(),
+            starRating: z.number().int().optional(),
+            reviewerLanguage: z.string().optional(),
+            device: z.string().optional(),
+            androidOsVersion: z.number().int().optional(),
+            appVersionCode: z.number().int().optional(),
+            appVersionName: z.string().optional()
+          }).optional()
+        })
+      ).optional()
     })
-  ).optional()
+  ).optional(),
+  tokenPagination: z.object({ nextPageToken: z.string().optional() }).optional()
 });
 var googlePlayConsoleResources = defineResources({
   apps: {
     shape: "entity",
     filterable: [],
-    description: "Android app the connector is syncing. One entity per configured packageName.",
-    endpoint: "GET /androidpublisher/v3/applications/{packageName}/listings",
+    description: "Android app the connector is syncing. One entity per configured packageName, derived from the connector config; the Play Store listing title is only reachable through an Android Publisher edit and is not fetched.",
     fields: [
       {
         name: "package_name",
         description: "Reverse-DNS application id (e.g. com.example.app)."
-      },
-      {
-        name: "title",
-        description: "Play Store listing title in the default language. Empty if the listing has not been fetched yet."
-      },
-      {
-        name: "default_language",
-        description: "Default language code (BCP-47) configured for the Play Store listings."
       }
-    ],
-    responses: { listings: publisherListingSchema }
+    ]
   },
   gplay_crash_rate_by_day: {
     shape: "metric",
@@ -332,7 +404,10 @@ var googlePlayConsoleResources = defineResources({
     granularity: "day",
     endpoint: "POST /v1beta1/apps/{packageName}/crashRateMetricSet:query",
     dimensions: [
-      { name: "date", description: "Calendar day of the metric sample (UTC)." },
+      {
+        name: "date",
+        description: "Calendar day of the metric sample (America/Los_Angeles, the only timezone the Reporting API supports for daily aggregation)."
+      },
       {
         name: "package_name",
         description: "Reverse-DNS application id this sample is reported against."
@@ -347,7 +422,10 @@ var googlePlayConsoleResources = defineResources({
     granularity: "day",
     endpoint: "POST /v1beta1/apps/{packageName}/anrRateMetricSet:query",
     dimensions: [
-      { name: "date", description: "Calendar day of the metric sample (UTC)." },
+      {
+        name: "date",
+        description: "Calendar day of the metric sample (America/Los_Angeles, the only timezone the Reporting API supports for daily aggregation)."
+      },
       {
         name: "package_name",
         description: "Reverse-DNS application id this sample is reported against."
@@ -355,35 +433,54 @@ var googlePlayConsoleResources = defineResources({
     ],
     responses: { anr_rate: metricSetSchema() }
   },
-  gplay_ratings_by_day: {
+  gplay_error_count_by_day: {
     shape: "metric",
-    description: "Daily average user rating and rating count from the Play Developer Reporting API.",
-    unit: "stars",
+    description: "Daily count of error reports (crashes + ANRs + handled errors) from the Play Developer Reporting API.",
+    unit: "reports",
     granularity: "day",
-    endpoint: "POST /v1beta1/apps/{packageName}/ratingsMetricSet:query",
+    endpoint: "POST /v1beta1/apps/{packageName}/errorCountMetricSet:query",
     dimensions: [
-      { name: "date", description: "Calendar day of the metric sample (UTC)." },
+      {
+        name: "date",
+        description: "Calendar day of the metric sample (America/Los_Angeles, the only timezone the Reporting API supports for daily aggregation)."
+      },
       {
         name: "package_name",
         description: "Reverse-DNS application id this sample is reported against."
       }
     ],
-    responses: { ratings: metricSetSchema() }
+    responses: { errors: metricSetSchema() }
   },
-  gplay_error_count_by_day: {
+  gplay_app_ratings: {
     shape: "metric",
-    description: "Daily count of error reports (crashes + ANRs + handled errors) from the Play Developer Reporting API.",
-    unit: "reports",
-    granularity: "day",
-    endpoint: "POST /v1beta1/apps/{packageName}/errorCountMetricSet:query",
+    description: "Rolling per-review star ratings sampled from the most-recent user reviews via the Android Publisher reviews API (default 200, configurable via reviewLimit). Each sample carries one review with its star rating (1-5) as the value.",
+    unit: "stars",
+    endpoint: "GET /androidpublisher/v3/applications/{packageName}/reviews",
+    notes: "Not the lifetime average shown on the Play Store. The reviews API only returns reviews from roughly the past week, so this is a rolling sample; average over a time window downstream for a smoothed rating.",
     dimensions: [
-      { name: "date", description: "Calendar day of the metric sample (UTC)." },
       {
         name: "package_name",
-        description: "Reverse-DNS application id this sample is reported against."
+        description: "Reverse-DNS application id this review was filed against."
+      },
+      { name: "review_id", description: "Unique identifier of the review." },
+      {
+        name: "reviewer_language",
+        description: "BCP-47 language code the review was written in."
+      },
+      {
+        name: "device",
+        description: "Codename of the device the review was filed from."
+      },
+      {
+        name: "app_version_name",
+        description: "App version name the reviewer was running."
+      },
+      {
+        name: "android_os_version",
+        description: "Android SDK version the reviewer was running."
       }
     ],
-    responses: { errors: metricSetSchema() }
+    responses: { reviews: reviewsResponseSchema }
   }
 });
 var id = "google-play-console";
@@ -396,7 +493,8 @@ var GooglePlayConsoleConnector = class _GooglePlayConsoleConnector extends BaseC
     return new _GooglePlayConsoleConnector(
       {
         packageName: parsed.packageName,
-        lookbackDays: parsed.lookbackDays
+        lookbackDays: parsed.lookbackDays,
+        reviewLimit: parsed.reviewLimit
       },
       {
         serviceAccountJson: parsed.serviceAccountJson
@@ -434,18 +532,6 @@ var GooglePlayConsoleConnector = class _GooglePlayConsoleConnector extends BaseC
     this.cachedToken = await this.fetchOAuthToken(url, body, signal);
     return this.cachedToken.token;
   }
-  async fetchListings(accessToken, signal) {
-    const url = `${PUBLISHER_BASE}/androidpublisher/v3/applications/${encodeURIComponent(this.settings.packageName)}/listings`;
-    const res = await this.get(url, {
-      resource: "listings",
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        "User-Agent": connectorUserAgent("google-play-console")
-      },
-      signal
-    });
-    return res.body;
-  }
   async runMetricQuery(accessToken, cfg, dateRange, pageToken, signal) {
     const url = `${REPORTING_BASE}/v1beta1/apps/${encodeURIComponent(this.settings.packageName)}/${cfg.metricSet}:query`;
     const [sy, sm, sd] = dateRange.startDate.split("-").map(Number);
@@ -458,13 +544,13 @@ var GooglePlayConsoleConnector = class _GooglePlayConsoleConnector extends BaseC
           year: sy,
           month: sm,
           day: sd,
-          timeZone: { id: "UTC" }
+          timeZone: { id: DAILY_TIME_ZONE }
         },
         endTime: {
           year: ey,
           month: em,
           day: ed,
-          timeZone: { id: "UTC" }
+          timeZone: { id: DAILY_TIME_ZONE }
         }
       }
     };
@@ -472,7 +558,7 @@ var GooglePlayConsoleConnector = class _GooglePlayConsoleConnector extends BaseC
       body["pageToken"] = pageToken;
     }
     const res = await this.post(url, {
-      resource: cfg.metricSet,
+      resource: cfg.responseTag,
       headers: {
         Authorization: `Bearer ${accessToken}`,
         "Content-Type": "application/json",
@@ -483,33 +569,58 @@ var GooglePlayConsoleConnector = class _GooglePlayConsoleConnector extends BaseC
     });
     return res.body;
   }
-  async syncApps(accessToken, storage, signal) {
-    let title = "";
-    let defaultLanguage = "";
-    try {
-      const listings = await this.fetchListings(accessToken, signal);
-      defaultLanguage = listings.defaultLanguage ?? "";
-      const list = listings.listings ?? [];
-      const def = list.find((l) => l.language === defaultLanguage) ?? list[0];
-      title = def?.title ?? "";
-    } catch (err) {
-      this.logger.warn(
-        "Failed to fetch Play Console listings; emitting app entity with empty title",
-        { error: err.message }
-      );
-    }
+  async syncApps(storage) {
     const entity = {
       type: "apps",
       id: this.settings.packageName,
       attributes: {
-        package_name: this.settings.packageName,
-        title,
-        default_language: defaultLanguage
+        package_name: this.settings.packageName
       },
       updated_at: Date.now()
     };
     await storage.entities([entity], { types: ["apps"] });
   }
+  async fetchReviews(accessToken, signal) {
+    const reviews = [];
+    const base = `${PUBLISHER_BASE}/androidpublisher/v3/applications/${encodeURIComponent(this.settings.packageName)}/reviews`;
+    let token = void 0;
+    for (let page = 0; page < MAX_REVIEW_PAGES; page++) {
+      signal?.throwIfAborted();
+      const params = new URLSearchParams({
+        maxResults: String(REVIEWS_PAGE_SIZE)
+      });
+      if (token) {
+        params.set("token", token);
+      }
+      const res = await this.get(
+        `${base}?${params.toString()}`,
+        {
+          resource: "reviews",
+          headers: {
+            Authorization: `Bearer ${accessToken}`,
+            "User-Agent": connectorUserAgent("google-play-console")
+          },
+          signal
+        }
+      );
+      reviews.push(...res.body.reviews ?? []);
+      token = res.body.tokenPagination?.nextPageToken;
+      if (!token) {
+        return reviews;
+      }
+    }
+    this.logger.warn(
+      `Stopped paginating Play Console reviews after ${MAX_REVIEW_PAGES} pages; the most-recent reviews are still ranked first but older reviews may be omitted`,
+      { packageName: this.settings.packageName }
+    );
+    return reviews;
+  }
+  async syncReviews(accessToken, storage, signal) {
+    const limit = this.settings.reviewLimit ?? DEFAULT_REVIEW_LIMIT;
+    const reviews = await this.fetchReviews(accessToken, signal);
+    const samples = reviews.map((review) => reviewToRatingSample(review, this.settings.packageName)).filter((s) => s !== null).sort((a, b) => b.ts - a.ts).slice(0, limit);
+    await storage.metrics(samples, { names: [GPLAY_APP_RATINGS_METRIC] });
+  }
   async drainMetricPhase(accessToken, cfg, dateRange, signal) {
     const rows = [];
     let pageToken = void 0;
@@ -542,17 +653,26 @@ var GooglePlayConsoleConnector = class _GooglePlayConsoleConnector extends BaseC
       }
       return accessToken;
     };
-    const resumeIdx = cursor ? PHASE_ORDER.indexOf(cursor.phase) : -1;
+    const phases = selectActivePhases(
+      (resource) => RESOURCE_TO_PHASE[resource] ?? "apps",
+      PHASE_ORDER,
+      options.resources ? [...options.resources] : void 0
+    );
+    const resumeIdx = cursor ? phases.indexOf(cursor.phase) : -1;
     const startIdx = resumeIdx >= 0 ? resumeIdx : 0;
-    for (let i = startIdx; i < PHASE_ORDER.length; i++) {
-      const phase = PHASE_ORDER[i];
+    for (let i = startIdx; i < phases.length; i++) {
+      const phase = phases[i];
       if (signal?.aborted) {
         return { done: false, cursor: { phase, dateRange } };
       }
       try {
         if (phase === "apps") {
+          await this.syncApps(storage);
+          continue;
+        }
+        if (phase === "reviews") {
           const token2 = await getToken(signal);
-          await this.syncApps(token2, storage, signal);
+          await this.syncReviews(token2, storage, signal);
           continue;
         }
         const cfg = METRIC_PHASE_CONFIGS[phase];