@frengki0707/google-cloud-clone 1.33.0

package/lib/utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/utils.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { TraceFlags } from '@opentelemetry/api';\nimport type { ReadableSpan, TimedEvent } from '@opentelemetry/sdk-trace-base';\nimport { resolveCurrentPrincipal } from './auth.js';\n\n/**\n * The maximum length (in characters) of a logged input or output.\n * This limit exists to align the logs with GCP logging size limits.\n * */\nconst MAX_LOG_CONTENT_CHARS = 128_000;\n\n/**\n * The maximum length (in characters) of a flow path.\n */\nconst MAX_PATH_CHARS = 4096;\n\nexport function extractOuterFlowNameFromPath(path: string) {\n  if (!path || path === '<unknown>') {\n    return '<unknown>';\n  }\n\n  const flowName = path.match('/{(.+),t:flow}+');\n  return flowName ? flowName[1] : '<unknown>';\n}\n\nexport function truncate(\n  text: string,\n  limit: number = MAX_LOG_CONTENT_CHARS\n): string {\n  return text ? text.substring(0, limit) : text;\n}\n\nexport function truncatePath(path: string) {\n  return truncate(path, MAX_PATH_CHARS);\n}\n\n/**\n * Extract first feature name from a path\n * e.g. for /{myFlow,t:flow}/{myStep,t:flowStep}/{googleai/gemini-pro,t:action,s:model}\n * returns \"myFlow\"\n */\nexport function extractOuterFeatureNameFromPath(path: string) {\n  if (!path || path === '<unknown>') {\n    return '<unknown>';\n  }\n  const first = path.split('/')[1];\n  const featureName = first?.match(\n    '{(.+),t:(flow|action|prompt|dotprompt|helper)'\n  );\n  return featureName ? featureName[1] : '<unknown>';\n}\n\nexport function extractErrorName(events: TimedEvent[]): string | undefined {\n  return events\n    .filter((event) => event.name === 'exception')\n    .map((event) => {\n      const attributes = event.attributes;\n      return attributes\n        ? truncate(attributes['exception.type'] as string, 1024)\n        : '<unknown>';\n    })\n    .at(0);\n}\n\nexport function extractErrorMessage(events: TimedEvent[]): string | undefined {\n  return events\n    .filter((event) => event.name === 'exception')\n    .map((event) => {\n      const attributes = event.attributes;\n      return attributes\n        ? truncate(attributes['exception.message'] as string, 4096)\n        : '<unknown>';\n    })\n    .at(0);\n}\n\nexport function extractErrorStack(events: TimedEvent[]): string | undefined {\n  return events\n    .filter((event) => event.name === 'exception')\n    .map((event) => {\n      const attributes = event.attributes;\n      return attributes\n        ? truncate(attributes['exception.stacktrace'] as string, 32_768)\n        : '<unknown>';\n    })\n    .at(0);\n}\n\nexport function createCommonLogAttributes(\n  span: ReadableSpan,\n  projectId?: string\n) {\n  const spanContext = span.spanContext();\n  const isSampled = !!(spanContext.traceFlags & TraceFlags.SAMPLED);\n  return {\n    'logging.googleapis.com/spanId': spanContext.spanId,\n    'logging.googleapis.com/trace': `projects/${projectId}/traces/${spanContext.traceId}`,\n    'logging.googleapis.com/trace_sampled': isSampled ? '1' : '0',\n  };\n}\n\nexport function requestDenied(\n  err: Error & {\n    code?: number;\n    statusDetails?: Record<string, any>[];\n  }\n) {\n  return err.code === 7;\n}\n\nexport function loggingDenied(\n  err: Error & {\n    code?: number;\n    statusDetails?: Record<string, any>[];\n  }\n) {\n  return (\n    requestDenied(err) &&\n    err.statusDetails?.some((details) => {\n      return details?.metadata?.permission === 'logging.logEntries.create';\n    })\n  );\n}\n\nexport function tracingDenied(\n  err: Error & {\n    code?: number;\n    statusDetails?: Record<string, any>[];\n  }\n) {\n  // Looks like we don't get status details like we do with logging\n  return requestDenied(err);\n}\n\nexport function metricsDenied(\n  err: Error & {\n    code?: number;\n    statusDetails?: Record<string, any>[];\n  }\n) {\n  // Looks like we don't get status details like we do with logging\n  return requestDenied(err);\n}\n\nexport async function permissionDeniedHelpText(role: string) {\n  const principal = await resolveCurrentPrincipal();\n  return `Add the role '${role}' to your Service Account in the IAM & Admin page on the Google Cloud console, or use the following command:\\n\\ngcloud projects add-iam-policy-binding ${principal.projectId ?? '${PROJECT_ID}'} \\\\\\n    --member=serviceAccount:${principal.serviceAccountEmail || '${SERVICE_ACCT}'} \\\\\\n    --role=${role}`;\n}\n\nexport async function loggingDeniedHelpText() {\n  return permissionDeniedHelpText('roles/logging.logWriter');\n}\n\nexport async function tracingDeniedHelpText() {\n  return permissionDeniedHelpText('roles/cloudtrace.agent');\n}\n\nexport async function metricsDeniedHelpText() {\n  return permissionDeniedHelpText('roles/monitoring.metricWriter');\n}\n"],"mappings":"AAgBA,SAAS,kBAAkB;AAE3B,SAAS,+BAA+B;AAMxC,MAAM,wBAAwB;AAK9B,MAAM,iBAAiB;AAEhB,SAAS,6BAA6B,MAAc;AACzD,MAAI,CAAC,QAAQ,SAAS,aAAa;AACjC,WAAO;AAAA,EACT;AAEA,QAAM,WAAW,KAAK,MAAM,iBAAiB;AAC7C,SAAO,WAAW,SAAS,CAAC,IAAI;AAClC;AAEO,SAAS,SACd,MACA,QAAgB,uBACR;AACR,SAAO,OAAO,KAAK,UAAU,GAAG,KAAK,IAAI;AAC3C;AAEO,SAAS,aAAa,MAAc;AACzC,SAAO,SAAS,MAAM,cAAc;AACtC;AAOO,SAAS,gCAAgC,MAAc;AAC5D,MAAI,CAAC,QAAQ,SAAS,aAAa;AACjC,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,KAAK,MAAM,GAAG,EAAE,CAAC;AAC/B,QAAM,cAAc,OAAO;AAAA,IACzB;AAAA,EACF;AACA,SAAO,cAAc,YAAY,CAAC,IAAI;AACxC;AAEO,SAAS,iBAAiB,QAA0C;AACzE,SAAO,OACJ,OAAO,CAAC,UAAU,MAAM,SAAS,WAAW,EAC5C,IAAI,CAAC,UAAU;AACd,UAAM,aAAa,MAAM;AACzB,WAAO,aACH,SAAS,WAAW,gBAAgB,GAAa,IAAI,IACrD;AAAA,EACN,CAAC,EACA,GAAG,CAAC;AACT;AAEO,SAAS,oBAAoB,QAA0C;AAC5E,SAAO,OACJ,OAAO,CAAC,UAAU,MAAM,SAAS,WAAW,EAC5C,IAAI,CAAC,UAAU;AACd,UAAM,aAAa,MAAM;AACzB,WAAO,aACH,SAAS,WAAW,mBAAmB,GAAa,IAAI,IACxD;AAAA,EACN,CAAC,EACA,GAAG,CAAC;AACT;AAEO,SAAS,kBAAkB,QAA0C;AAC1E,SAAO,OACJ,OAAO,CAAC,UAAU,MAAM,SAAS,WAAW,EAC5C,IAAI,CAAC,UAAU;AACd,UAAM,aAAa,MAAM;AACzB,WAAO,aACH,SAAS,WAAW,sBAAsB,GAAa,KAAM,IAC7D;AAAA,EACN,CAAC,EACA,GAAG,CAAC;AACT;AAEO,SAAS,0BACd,MACA,WACA;AACA,QAAM,cAAc,KAAK,YAAY;AACrC,QAAM,YAAY,CAAC,EAAE,YAAY,aAAa,WAAW;AACzD,SAAO;AAAA,IACL,iCAAiC,YAAY;AAAA,IAC7C,gCAAgC,YAAY,SAAS,WAAW,YAAY,OAAO;AAAA,IACnF,wCAAwC,YAAY,MAAM;AAAA,EAC5D;AACF;AAEO,SAAS,cACd,KAIA;AACA,SAAO,IAAI,SAAS;AACtB;AAEO,SAAS,cACd,KAIA;AACA,SACE,cAAc,GAAG,KACjB,IAAI,eAAe,KAAK,CAAC,YAAY;AACnC,WAAO,SAAS,UAAU,eAAe;AAAA,EAC3C,CAAC;AAEL;AAEO,SAAS,cACd,KAIA;AAEA,SAAO,cAAc,GAAG;AAC1B;AAEO,SAAS,cACd,KAIA;AAEA,SAAO,cAAc,GAAG;AAC1B;AAEA,eAAsB,yBAAyB,MAAc;AAC3D,QAAM,YAAY,MAAM,wBAAwB;AAChD,SAAO,iBAAiB,IAAI;AAAA;AAAA,yCAA0J,UAAU,aAAa,eAAe;AAAA,8BAAoC,UAAU,uBAAuB,iBAAiB;AAAA,aAAmB,IAAI;AAC3U;AAEA,eAAsB,wBAAwB;AAC5C,SAAO,yBAAyB,yBAAyB;AAC3D;AAEA,eAAsB,wBAAwB;AAC5C,SAAO,yBAAyB,wBAAwB;AAC1D;AAEA,eAAsB,wBAAwB;AAC5C,SAAO,yBAAyB,+BAA+B;AACjE;","names":[]}

package/package.json

@@ -0,0 +1,89 @@
+{
+  "name": "@frengki0707/google-cloud-clone",
+  "description": "Genkit AI framework plugin for Google Cloud Platform including Firestore trace/state store and deployment helpers for Cloud Functions for Firebase.",
+  "keywords": [
+    "genkit",
+    "genkit-plugin",
+    "genkit-telemetry",
+    "genkit-deploy",
+    "genkit-flow",
+    "google cloud",
+    "google cloud platform",
+    "ai",
+    "genai",
+    "generative-ai"
+  ],
+  "version": "1.33.0",
+  "type": "commonjs",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/genkit-ai/genkit.git",
+    "directory": "js/plugins/google-cloud"
+  },
+  "author": "genkit",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@google-cloud/logging-winston": "^6.0.0",
+    "@google-cloud/modelarmor": "^0.4.1",
+    "@google-cloud/opentelemetry-cloud-monitoring-exporter": "^0.19.0",
+    "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.4.1",
+    "@google-cloud/opentelemetry-resource-util": "^2.4.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.49.1",
+    "@opentelemetry/core": "~1.25.0",
+    "@opentelemetry/instrumentation": "^0.52.0",
+    "@opentelemetry/instrumentation-pino": "^0.41.0",
+    "@opentelemetry/instrumentation-winston": "^0.39.0",
+    "@opentelemetry/resources": "~1.25.0",
+    "@opentelemetry/sdk-metrics": "~1.25.0",
+    "@opentelemetry/sdk-node": "^0.52.0",
+    "@opentelemetry/sdk-trace-base": "~1.25.0",
+    "google-auth-library": "^9.6.3",
+    "node-fetch": "^3.3.2",
+    "winston": "^3.12.0"
+  },
+  "peerDependencies": {
+    "genkit": "^1.33.0"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@types/node": "^20.11.16",
+    "jest": "^29.7.0",
+    "npm-run-all": "^4.1.5",
+    "rimraf": "^6.0.1",
+    "ts-jest": "^29.1.2",
+    "tsup": "^8.3.5",
+    "tsx": "^4.19.2",
+    "typescript": "^5.9.3"
+  },
+  "types": "./lib/index.d.ts",
+  "exports": {
+    ".": {
+      "require": "./lib/index.js",
+      "import": "./lib/index.mjs",
+      "types": "./lib/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./model-armor": {
+      "require": "./lib/model-armor.js",
+      "import": "./lib/model-armor.mjs",
+      "types": "./lib/model-armor.d.ts",
+      "default": "./lib/model-armor.js"
+    }
+  },
+  "typesVersions": {
+    "*": {
+      "model-armor": [
+        "lib/model-armor"
+      ]
+    }
+  },
+  "scripts": {
+    "check": "tsc",
+    "compile": "tsup-node",
+    "build:clean": "rimraf ./lib",
+    "build": "npm-run-all build:clean check compile",
+    "build:watch": "tsup-node --watch",
+    "test": "node node_modules/jest/bin/jest --runInBand --verbose"
+  }
+}

package/src/auth.ts

@@ -0,0 +1,89 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { logger } from 'genkit/logging';
+import { GoogleAuth, auth, type CredentialBody } from 'google-auth-library';
+import type { GcpPrincipal, GcpTelemetryConfig } from './types.js';
+
+/**
+ * Allows Google Cloud credentials to be to passed in "raw" as an environment
+ * variable. This is helpful in environments where the developer has limited
+ * ability to configure their compute environment, but does have the ablilty to
+ * set environment variables.
+ *
+ * This is different from the GOOGLE_APPLICATION_CREDENTIALS used by ADC, which
+ * represents a path to a credential file on disk. In *most* cases, even for
+ * 3rd party cloud providers, developers *should* attempt to use ADC, which
+ * searches for credential files in standard locations, before using this
+ * method.
+ *
+ * See also: https://github.com/googleapis/google-auth-library-nodejs?tab=readme-ov-file#loading-credentials-from-environment-variables
+ */
+export async function credentialsFromEnvironment(): Promise<
+  Partial<GcpTelemetryConfig>
+> {
+  let authClient: GoogleAuth;
+  const options: Partial<GcpTelemetryConfig> = {};
+
+  if (process.env.GCLOUD_SERVICE_ACCOUNT_CREDS) {
+    logger.debug('Retrieving credentials from GCLOUD_SERVICE_ACCOUNT_CREDS');
+    const serviceAccountCreds = JSON.parse(
+      process.env.GCLOUD_SERVICE_ACCOUNT_CREDS
+    );
+    const authOptions = { credentials: serviceAccountCreds };
+    authClient = new GoogleAuth(authOptions);
+    options.credentials = await authClient.getCredentials();
+  } else {
+    authClient = new GoogleAuth();
+  }
+  try {
+    const projectId = await authClient.getProjectId();
+    if (projectId && projectId.length > 0) {
+      options.projectId = projectId;
+    }
+  } catch (error) {
+    logger.warn(error);
+  }
+  return options;
+}
+
+/**
+ * Resolve the currently configured principal, either from the Genkit specific
+ * GCLOUD_SERVICE_ACCOUNT_CREDS environment variable, or from ADC.
+ *
+ * Since the Google Cloud Telemetry Exporter will discover credentials on its
+ * own, we don't immediately have access to the current principal. This method
+ * can be handy to get access to the current credential for logging debugging
+ * information or other purposes.
+ **/
+export async function resolveCurrentPrincipal(): Promise<GcpPrincipal> {
+  const envCredentials = await credentialsFromEnvironment();
+  let adcCredentials = {} as CredentialBody;
+  try {
+    adcCredentials = await auth.getCredentials();
+  } catch (e) {
+    logger.debug('Could not retrieve client_email from ADC.');
+  }
+
+  // TODO(michaeldoyle): How to look up if the user provided credentials in the
+  // plugin config (i.e. GcpTelemetryOptions)
+  const serviceAccountEmail =
+    envCredentials.credentials?.client_email ?? adcCredentials.client_email;
+
+  return {
+    projectId: envCredentials.projectId,
+    serviceAccountEmail,
+  };
+}

package/src/gcpLogger.ts

@@ -0,0 +1,124 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LoggingWinston } from '@google-cloud/logging-winston';
+import { getCurrentEnv } from 'genkit';
+import { logger } from 'genkit/logging';
+import type { Writable } from 'stream';
+import type { GcpTelemetryConfig } from './types.js';
+import { loggingDenied, loggingDeniedHelpText } from './utils.js';
+
+/**
+ * Additional streams for writing log data to. Useful for unit testing.
+ */
+let additionalStream: Writable;
+let useJsonFormatOverride = false;
+
+/**
+ * Provides a logger for exporting Genkit debug logs to GCP Cloud
+ * logs.
+ */
+export class GcpLogger {
+  constructor(private readonly config: GcpTelemetryConfig) {}
+
+  async getLogger(env: string) {
+    // Dynamically importing winston here more strictly controls
+    // the import order relative to registering instrumentation
+    // with OpenTelemetry. Incorrect import order will trigger
+    // an internal OT warning and will result in logs not being
+    // associated with correct spans/traces.
+    const winston = await import('winston');
+
+    const format =
+      useJsonFormatOverride || this.shouldExport(env)
+        ? { format: winston.format.json() }
+        : {
+            format: winston.format.printf((info): string => {
+              return `[${info.level}] ${info.message}`;
+            }),
+          };
+
+    const transports: any[] = [];
+    transports.push(
+      this.shouldExport(env)
+        ? new LoggingWinston({
+            labels: { module: 'genkit' },
+            prefix: 'genkit',
+            logName: 'genkit_log',
+            projectId: this.config.projectId,
+            credentials: this.config.credentials,
+            autoRetry: true,
+            defaultCallback: await this.getErrorHandler(),
+          })
+        : new winston.transports.Console()
+    );
+    if (additionalStream) {
+      transports.push(
+        new winston.transports.Stream({ stream: additionalStream })
+      );
+    }
+    return winston.createLogger({
+      transports: transports,
+      ...format,
+      exceptionHandlers: [new winston.transports.Console()],
+    });
+  }
+
+  private async getErrorHandler(): Promise<(err: Error | null) => void> {
+    // only log the first time
+    let instructionsLogged = false;
+    const helpInstructions = await loggingDeniedHelpText();
+
+    return async (err: Error | null) => {
+      // Use the defaultLogger so that logs don't get swallowed by
+      // the open telemetry exporter
+      const defaultLogger = logger.defaultLogger;
+      if (err && loggingDenied(err)) {
+        if (!instructionsLogged) {
+          instructionsLogged = true;
+          defaultLogger.error(
+            `Unable to send logs to Google Cloud: ${err.message}\n\n${helpInstructions}\n`
+          );
+        }
+      } else if (err) {
+        defaultLogger.error(`Unable to send logs to Google Cloud: ${err}`);
+      }
+
+      if (err) {
+        // Assume the logger is compromised, and we need a new one
+        // Reinitialize the genkit logger with a new instance with the same config
+        logger.init(
+          await new GcpLogger(this.config).getLogger(getCurrentEnv())
+        );
+        defaultLogger.info('Initialized a new GcpLogger.');
+      }
+    };
+  }
+
+  private shouldExport(env?: string) {
+    return this.config.export;
+  }
+}
+
+/** @hidden */
+export function __addTransportStreamForTesting(stream: Writable) {
+  additionalStream = stream;
+}
+
+/** @hidden */
+export function __useJsonFormatForTesting() {
+  useJsonFormatOverride = true;
+}