@frengki0707/google-cloud-clone 1.33.0

package/src/telemetry/path.ts

@@ -0,0 +1,111 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ValueType } from '@opentelemetry/api';
+import { hrTimeDuration, hrTimeToMilliseconds } from '@opentelemetry/core';
+import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
+import { GENKIT_VERSION } from 'genkit';
+import { logger } from 'genkit/logging';
+import { toDisplayPath } from 'genkit/tracing';
+import {
+  MetricCounter,
+  MetricHistogram,
+  internalMetricNamespaceWrap,
+  type Telemetry,
+} from '../metrics.js';
+import {
+  createCommonLogAttributes,
+  extractErrorMessage,
+  extractErrorName,
+  extractErrorStack,
+  extractOuterFeatureNameFromPath,
+  truncatePath,
+} from '../utils.js';
+
+class PathsTelemetry implements Telemetry {
+  /**
+   * Wraps the declared metrics in a Genkit-specific, internal namespace.
+   */
+  private _N = internalMetricNamespaceWrap.bind(null, 'feature');
+
+  private pathCounter = new MetricCounter(this._N('path/requests'), {
+    description: 'Tracks unique flow paths per flow.',
+    valueType: ValueType.INT,
+  });
+
+  private pathLatencies = new MetricHistogram(this._N('path/latency'), {
+    description: 'Latencies per flow path.',
+    ValueType: ValueType.DOUBLE,
+    unit: 'ms',
+  });
+
+  tick(
+    span: ReadableSpan,
+    logInputAndOutput: boolean,
+    projectId?: string
+  ): void {
+    const attributes = span.attributes;
+
+    const path = attributes['genkit:path'] as string;
+
+    const isFailureSource = !!span.attributes['genkit:isFailureSource'];
+    const state = attributes['genkit:state'] as string;
+
+    if (!path || !isFailureSource || state !== 'error') {
+      // Only tick metrics for failing, leaf spans.
+      return;
+    }
+
+    const sessionId = attributes['genkit:sessionId'] as string;
+    const threadName = attributes['genkit:threadName'] as string;
+
+    const errorName = extractErrorName(span.events) || '<unknown>';
+    const errorMessage = extractErrorMessage(span.events) || '<unknown>';
+    const errorStack = extractErrorStack(span.events) || '';
+
+    const latency = hrTimeToMilliseconds(
+      hrTimeDuration(span.startTime, span.endTime)
+    );
+
+    const pathDimensions = {
+      featureName: extractOuterFeatureNameFromPath(path),
+      status: 'failure',
+      error: errorName,
+      path: path,
+      source: 'ts',
+      sourceVersion: GENKIT_VERSION,
+    };
+    this.pathCounter.add(1, pathDimensions);
+    this.pathLatencies.record(latency, pathDimensions);
+
+    const displayPath = truncatePath(toDisplayPath(path));
+    logger.logStructuredError(`Error[${displayPath}, ${errorName}]`, {
+      ...createCommonLogAttributes(span, projectId),
+      path: displayPath,
+      qualifiedPath: path,
+      name: errorName,
+      message: errorMessage,
+      stack: errorStack,
+      source: 'ts',
+      sourceVersion: GENKIT_VERSION,
+      sessionId,
+      threadName,
+    });
+  }
+}
+
+const pathsTelemetry = new PathsTelemetry();
+export { pathsTelemetry };

package/src/types.ts

@@ -0,0 +1,133 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { InstrumentationConfigMap } from '@opentelemetry/auto-instrumentations-node';
+import type { Instrumentation } from '@opentelemetry/instrumentation';
+import type { Sampler } from '@opentelemetry/sdk-trace-base';
+import type { JWTInput } from 'google-auth-library';
+
+/** Configuration options for the Google Cloud plugin. */
+export interface GcpTelemetryConfigOptions {
+  /**
+   * Google Cloud Project ID. If provided, will take precedence over the
+   * projectId inferred from the application credential and/or environment.
+   * Required when providing an external credential (e.g. Workload Identity
+   * Federation.)
+   */
+  projectId?: string;
+
+  /**
+   * Credentials for authenticating with Google Cloud. Primarily intended for
+   * use in environments outside of GCP. On GCP credentials will typically be
+   * inferred from the environment via Application Default Credentials (ADC).
+   */
+  credentials?: JWTInput;
+
+  /**
+   * OpenTelemetry sampler; controls the number of traces collected and exported
+   * to Google Cloud. Defaults to AlwaysOnSampler, which will collect and export
+   * all traces.
+   *
+   * There are four built-in samplers to choose from:
+   *
+   * - {@link https://github.com/open-telemetry/opentelemetry-js/blob/main/packages/opentelemetry-sdk-trace-base/src/sampler/AlwaysOnSampler.ts | AlwaysOnSampler} - samples all traces
+   * - {@link https://github.com/open-telemetry/opentelemetry-js/blob/main/packages/opentelemetry-sdk-trace-base/src/sampler/AlwaysOffSampler.ts | AlwaysOffSampler} - samples no traces
+   * - {@link https://github.com/open-telemetry/opentelemetry-js/blob/main/packages/opentelemetry-sdk-trace-base/src/sampler/ParentBasedSampler.ts | ParentBasedSampler} - samples based on parent span
+   * - {@link https://github.com/open-telemetry/opentelemetry-js/blob/main/packages/opentelemetry-sdk-trace-base/src/sampler/TraceIdRatioBasedSampler.ts | TraceIdRatioBasedSampler} - samples a configurable percentage of traces
+   */
+  sampler?: Sampler;
+
+  /**
+   * Enabled by default, OpenTelemetry will automatically collect telemetry for
+   * popular libraries via auto instrumentations without any additional code
+   * or configuration. All available instrumentations will be collected, unless
+   * otherwise specified via {@link autoInstrumentationConfig}.
+   *
+   * @see https://opentelemetry.io/docs/zero-code/js/
+   */
+  autoInstrumentation?: boolean;
+
+  /**
+   * Map of auto instrumentations and their configuration options. Available
+   * options will vary by instrumentation.
+   *
+   * @see https://opentelemetry.io/docs/zero-code/js/
+   */
+  autoInstrumentationConfig?: InstrumentationConfigMap;
+
+  /**
+   * Additional OpenTelemetry instrumentations to include, beyond those
+   * provided by auto instrumentations.
+   */
+  instrumentations?: Instrumentation[];
+
+  /**
+   * Metrics export interval in milliseconds; Google Cloud requires a minimum
+   * value of 5000ms.
+   */
+  metricExportIntervalMillis?: number;
+
+  /**
+   * Timeout for the metrics export in milliseconds.
+   */
+  metricExportTimeoutMillis?: number;
+
+  /**
+   * If set to true, metrics will not be exported to Google Cloud. Traces and
+   * logs may still be exported.
+   */
+  disableMetrics?: boolean;
+
+  /**
+   * If set to true, traces will not be exported to Google Cloud. Metrics and
+   * logs may still be exported.
+   */
+  disableTraces?: boolean;
+
+  /**
+   * If set to true, input and output logs will not be collected.
+   */
+  disableLoggingInputAndOutput?: boolean;
+
+  /**
+   * If set to true, telemetry data will be exported in the Genkit `dev`
+   * environment. Useful for local testing and troubleshooting; default is
+   * false.
+   */
+  forceDevExport?: boolean;
+}
+
+/** Internal telemetry configuration. */
+export interface GcpTelemetryConfig {
+  projectId?: string;
+  credentials?: JWTInput;
+
+  sampler: Sampler;
+  autoInstrumentation: boolean;
+  autoInstrumentationConfig: InstrumentationConfigMap;
+  metricExportIntervalMillis: number;
+  metricExportTimeoutMillis: number;
+  instrumentations: Instrumentation[];
+  disableMetrics: boolean;
+  disableTraces: boolean;
+  exportInputAndOutput: boolean;
+  export: boolean;
+}
+
+export interface GcpPrincipal {
+  projectId?: string;
+  serviceAccountEmail?: string;
+}

package/src/utils.ts

@@ -0,0 +1,175 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TraceFlags } from '@opentelemetry/api';
+import type { ReadableSpan, TimedEvent } from '@opentelemetry/sdk-trace-base';
+import { resolveCurrentPrincipal } from './auth.js';
+
+/**
+ * The maximum length (in characters) of a logged input or output.
+ * This limit exists to align the logs with GCP logging size limits.
+ * */
+const MAX_LOG_CONTENT_CHARS = 128_000;
+
+/**
+ * The maximum length (in characters) of a flow path.
+ */
+const MAX_PATH_CHARS = 4096;
+
+export function extractOuterFlowNameFromPath(path: string) {
+  if (!path || path === '<unknown>') {
+    return '<unknown>';
+  }
+
+  const flowName = path.match('/{(.+),t:flow}+');
+  return flowName ? flowName[1] : '<unknown>';
+}
+
+export function truncate(
+  text: string,
+  limit: number = MAX_LOG_CONTENT_CHARS
+): string {
+  return text ? text.substring(0, limit) : text;
+}
+
+export function truncatePath(path: string) {
+  return truncate(path, MAX_PATH_CHARS);
+}
+
+/**
+ * Extract first feature name from a path
+ * e.g. for /{myFlow,t:flow}/{myStep,t:flowStep}/{googleai/gemini-pro,t:action,s:model}
+ * returns "myFlow"
+ */
+export function extractOuterFeatureNameFromPath(path: string) {
+  if (!path || path === '<unknown>') {
+    return '<unknown>';
+  }
+  const first = path.split('/')[1];
+  const featureName = first?.match(
+    '{(.+),t:(flow|action|prompt|dotprompt|helper)'
+  );
+  return featureName ? featureName[1] : '<unknown>';
+}
+
+export function extractErrorName(events: TimedEvent[]): string | undefined {
+  return events
+    .filter((event) => event.name === 'exception')
+    .map((event) => {
+      const attributes = event.attributes;
+      return attributes
+        ? truncate(attributes['exception.type'] as string, 1024)
+        : '<unknown>';
+    })
+    .at(0);
+}
+
+export function extractErrorMessage(events: TimedEvent[]): string | undefined {
+  return events
+    .filter((event) => event.name === 'exception')
+    .map((event) => {
+      const attributes = event.attributes;
+      return attributes
+        ? truncate(attributes['exception.message'] as string, 4096)
+        : '<unknown>';
+    })
+    .at(0);
+}
+
+export function extractErrorStack(events: TimedEvent[]): string | undefined {
+  return events
+    .filter((event) => event.name === 'exception')
+    .map((event) => {
+      const attributes = event.attributes;
+      return attributes
+        ? truncate(attributes['exception.stacktrace'] as string, 32_768)
+        : '<unknown>';
+    })
+    .at(0);
+}
+
+export function createCommonLogAttributes(
+  span: ReadableSpan,
+  projectId?: string
+) {
+  const spanContext = span.spanContext();
+  const isSampled = !!(spanContext.traceFlags & TraceFlags.SAMPLED);
+  return {
+    'logging.googleapis.com/spanId': spanContext.spanId,
+    'logging.googleapis.com/trace': `projects/${projectId}/traces/${spanContext.traceId}`,
+    'logging.googleapis.com/trace_sampled': isSampled ? '1' : '0',
+  };
+}
+
+export function requestDenied(
+  err: Error & {
+    code?: number;
+    statusDetails?: Record<string, any>[];
+  }
+) {
+  return err.code === 7;
+}
+
+export function loggingDenied(
+  err: Error & {
+    code?: number;
+    statusDetails?: Record<string, any>[];
+  }
+) {
+  return (
+    requestDenied(err) &&
+    err.statusDetails?.some((details) => {
+      return details?.metadata?.permission === 'logging.logEntries.create';
+    })
+  );
+}
+
+export function tracingDenied(
+  err: Error & {
+    code?: number;
+    statusDetails?: Record<string, any>[];
+  }
+) {
+  // Looks like we don't get status details like we do with logging
+  return requestDenied(err);
+}
+
+export function metricsDenied(
+  err: Error & {
+    code?: number;
+    statusDetails?: Record<string, any>[];
+  }
+) {
+  // Looks like we don't get status details like we do with logging
+  return requestDenied(err);
+}
+
+export async function permissionDeniedHelpText(role: string) {
+  const principal = await resolveCurrentPrincipal();
+  return `Add the role '${role}' to your Service Account in the IAM & Admin page on the Google Cloud console, or use the following command:\n\ngcloud projects add-iam-policy-binding ${principal.projectId ?? '${PROJECT_ID}'} \\\n    --member=serviceAccount:${principal.serviceAccountEmail || '${SERVICE_ACCT}'} \\\n    --role=${role}`;
+}
+
+export async function loggingDeniedHelpText() {
+  return permissionDeniedHelpText('roles/logging.logWriter');
+}
+
+export async function tracingDeniedHelpText() {
+  return permissionDeniedHelpText('roles/cloudtrace.agent');
+}
+
+export async function metricsDeniedHelpText() {
+  return permissionDeniedHelpText('roles/monitoring.metricWriter');
+}

package/tests/logs_no_input_output_test.ts

@@ -0,0 +1,267 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  it,
+  jest,
+} from '@jest/globals';
+import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
+import * as assert from 'assert';
+import { genkit, z, type GenerateResponseData, type Genkit } from 'genkit';
+import { Writable } from 'stream';
+import {
+  __addTransportStreamForTesting,
+  __forceFlushSpansForTesting,
+  __getSpanExporterForTesting,
+  enableGoogleCloudTelemetry,
+} from '../src/index.js';
+
+jest.mock('../src/auth.js', () => {
+  const original = jest.requireActual('../src/auth.js');
+  return {
+    ...(original || {}),
+    resolveCurrentPrincipal: jest.fn().mockImplementation(() => {
+      return Promise.resolve({
+        projectId: 'test',
+        serviceAccountEmail: 'test@test.com',
+      });
+    }),
+    credentialsFromEnvironment: jest.fn().mockImplementation(() => {
+      return Promise.resolve({
+        projectId: 'test',
+        credentials: {
+          client_email: 'test@genkit.com',
+          private_key: '-----BEGIN PRIVATE KEY-----',
+        },
+      });
+    }),
+  };
+});
+
+describe('GoogleCloudLogs no I/O', () => {
+  let logLines = '';
+  const logStream = new Writable();
+  logStream._write = (chunk, encoding, next) => {
+    logLines = logLines += chunk.toString();
+    next();
+  };
+
+  let ai: Genkit;
+
+  beforeAll(async () => {
+    process.env.GCLOUD_PROJECT = 'test';
+    process.env.GENKIT_ENV = 'dev';
+    __addTransportStreamForTesting(logStream);
+    await enableGoogleCloudTelemetry({
+      projectId: 'test',
+      forceDevExport: false,
+      metricExportIntervalMillis: 100,
+      metricExportTimeoutMillis: 100,
+      disableLoggingInputAndOutput: true,
+    });
+    ai = genkit({});
+    // Wait for the telemetry plugin to be initialized
+    await waitForLogsInit(ai, logLines);
+  });
+  beforeEach(async () => {
+    logLines = '';
+    __getSpanExporterForTesting().reset();
+  });
+  afterAll(async () => {
+    await ai.stopServers();
+  });
+
+  it('writes error logs', async () => {
+    const testFlow = createFlow(ai, 'testFlow', async () => {
+      const nothing: { missing?: any } = { missing: 1 };
+      delete nothing.missing;
+      return nothing.missing.explode;
+    });
+
+    await assert.rejects(async () => {
+      await testFlow();
+    });
+
+    await getExportedSpans();
+
+    const logMessages = await getLogs(1, 100, logLines);
+    assert.equal(
+      logMessages.includes(
+        "[error] Error[testFlow, TypeError] Cannot read properties of undefined (reading 'explode')"
+      ),
+      true
+    );
+  }, 10000); //timeout
+
+  it('writes generate logs', async () => {
+    const testModel = createModel(ai, 'testModel', async () => {
+      return {
+        message: {
+          role: 'user',
+          content: [
+            {
+              text: 'response',
+            },
+          ],
+        },
+        finishReason: 'stop',
+        usage: {
+          inputTokens: 10,
+          outputTokens: 14,
+          inputCharacters: 8,
+          outputCharacters: 16,
+          inputImages: 1,
+          outputImages: 3,
+        },
+      };
+    });
+    const testFlow = createFlowWithInput(ai, 'testFlow', async (input) => {
+      return await ai.run('sub1', async () => {
+        return await ai.run('sub2', async () => {
+          return await ai.generate({
+            model: testModel,
+            prompt: `${input} prompt`,
+            config: {
+              temperature: 1.0,
+              topK: 3,
+              topP: 5,
+              maxOutputTokens: 7,
+            },
+          });
+        });
+      });
+    });
+
+    await testFlow('test');
+
+    await getExportedSpans();
+
+    const logMessages = await getLogs(1, 100, logLines);
+    assert.equal(
+      logMessages.includes(
+        '[info] Config[testFlow > sub1 > sub2 > generate > testModel, testModel]'
+      ),
+      true
+    );
+    assert.equal(
+      logMessages.includes(
+        '[info] Input[testFlow > sub1 > sub2 > generate > testModel, testModel]'
+      ),
+      false
+    );
+    assert.equal(
+      logMessages.includes(
+        '[info] Output[testFlow > sub1 > sub2 > generate > testModel, testModel]'
+      ),
+      false
+    );
+    assert.equal(
+      logMessages.includes('[info] Input[testFlow, testModel]'),
+      false
+    );
+    assert.equal(
+      logMessages.includes('[info] Output[testFlow, testModel]'),
+      false
+    );
+  });
+});
+
+/** Helper to create a flow with no inputs or outputs */
+function createFlow(
+  ai: Genkit,
+  name: string,
+  fn: () => Promise<any> = async () => {}
+) {
+  return ai.defineFlow(
+    {
+      name,
+      inputSchema: z.void(),
+      outputSchema: z.void(),
+    },
+    fn
+  );
+}
+
+function createFlowWithInput(
+  ai: Genkit,
+  name: string,
+  fn: (input: string) => Promise<any>
+) {
+  return ai.defineFlow(
+    {
+      name,
+      inputSchema: z.string(),
+      outputSchema: z.any(),
+    },
+    fn
+  );
+}
+
+/**
+ * Helper to create a model that returns the value produced by the given
+ * response function.
+ */
+function createModel(
+  genkit: Genkit,
+  name: string,
+  respFn: () => Promise<GenerateResponseData>
+) {
+  return genkit.defineModel({ name }, (req) => respFn());
+}
+
+async function waitForLogsInit(genkit: Genkit, logLines: any) {
+  await import('winston');
+  const testFlow = createFlow(genkit, 'testFlow');
+  await testFlow();
+  await getLogs(1, 100, logLines);
+}
+
+async function getLogs(
+  logCount: number,
+  maxAttempts: number,
+  logLines: string
+): Promise<string[]> {
+  var attempts = 0;
+  while (attempts++ < maxAttempts) {
+    await new Promise((resolve) => setTimeout(resolve, 100));
+    const found = logLines
+      .trim()
+      .split('\n')
+      .map((l) => l.trim());
+    if (found.length >= logCount) {
+      return found;
+    }
+  }
+  assert.fail(`Waiting for logs, but none have been written.`);
+}
+
+/** Polls the in memory metric exporter until the genkit scope is found. */
+async function getExportedSpans(maxAttempts = 200): Promise<ReadableSpan[]> {
+  __forceFlushSpansForTesting();
+  var attempts = 0;
+  while (attempts++ < maxAttempts) {
+    await new Promise((resolve) => setTimeout(resolve, 50));
+    const found = __getSpanExporterForTesting().getFinishedSpans();
+    if (found.length > 0) {
+      return found;
+    }
+  }
+  assert.fail(`Timed out while waiting for spans to be exported.`);
+}