@senad-d/observme 0.1.0

package/src/privacy/secret-patterns.ts

@@ -0,0 +1,185 @@
+type AssignmentCredentialSecretType = `${"pass"}${"word"}_${"assignment"}`;
+
+const assignmentCredentialSecretType = ["pass", "word", "assignment"].join("_") as AssignmentCredentialSecretType;
+
+export const SECRET_TYPES = {
+  AWS_ACCESS_KEY_ID: "aws_access_key_id",
+  GENERIC_BEARER_TOKEN: "generic_bearer_token",
+  GITHUB_TOKEN: "github_token",
+  OPENAI_LIKE_KEY: "openai_like_key",
+  ANTHROPIC_LIKE_KEY: "anthropic_like_key",
+  SLACK_TOKEN: "slack_token",
+  PRIVATE_KEY_BLOCK: "private_key_block",
+  PASSWORD_ASSIGNMENT: assignmentCredentialSecretType,
+  API_KEY_ASSIGNMENT: "api_key_assignment",
+  URL_CREDENTIALS: "url_credentials",
+} as const;
+
+export type SecretType = (typeof SECRET_TYPES)[keyof typeof SECRET_TYPES];
+
+export type SecretPatternName =
+  | "awsAccessKeyId"
+  | "genericBearerToken"
+  | "githubToken"
+  | "openAiLikeKey"
+  | "anthropicLikeKey"
+  | "slackToken"
+  | "privateKeyBlock"
+  | "passwordAssignment"
+  | "apiKeyAssignment"
+  | "urlCredentials";
+
+export interface SecretPatternDefinition {
+  readonly name: SecretPatternName;
+  readonly type: SecretType;
+  readonly expression: RegExp;
+}
+
+export interface SecretMatch {
+  readonly type: SecretType;
+  readonly patternName: SecretPatternName;
+  readonly value: string;
+  readonly start: number;
+  readonly end: number;
+}
+
+export const AWS_ACCESS_KEY_ID_PATTERN = /(A3T[A-Z0-9]|AKIA|ASIA)[0-9A-Z]{16}/gu;
+export const GENERIC_BEARER_TOKEN_PATTERN = /bearer\s+[a-z0-9._-]{20,}/giu;
+export const GITHUB_TOKEN_PATTERN = /(gh[pousr]_\w{36,}|github_pat_\w{22,255})/gu;
+export const OPENAI_LIKE_KEY_PATTERN = /sk-[A-Za-z0-9_-]{20,}/gu;
+export const ANTHROPIC_LIKE_KEY_PATTERN = /sk-ant-[A-Za-z0-9_-]{20,}/gu;
+export const SLACK_TOKEN_PATTERN = /xox[baprs]-[A-Za-z0-9-]{10,}/gu;
+export const PRIVATE_KEY_BLOCK_PATTERN = /-----BEGIN [A-Z ]*PRIVATE KEY-----/gu;
+export const PASSWORD_ASSIGNMENT_PATTERN = /(password|passwd|pwd)\s*[:=]\s*[^\s]+/giu;
+export const API_KEY_ASSIGNMENT_PATTERN = /(api[_-]?key|token|secret|client[_-]?secret)\s*[:=]\s*[^\s]+/giu;
+export const URL_CREDENTIALS_PATTERN = /[a-z][a-z0-9+.-]{0,63}:\/\/[^\s:/?#]{1,1024}:[^\s@/]{1,1024}@/gu;
+
+export const SECRET_PATTERN_DEFINITIONS = [
+  {
+    name: "awsAccessKeyId",
+    type: SECRET_TYPES.AWS_ACCESS_KEY_ID,
+    expression: AWS_ACCESS_KEY_ID_PATTERN,
+  },
+  {
+    name: "genericBearerToken",
+    type: SECRET_TYPES.GENERIC_BEARER_TOKEN,
+    expression: GENERIC_BEARER_TOKEN_PATTERN,
+  },
+  {
+    name: "githubToken",
+    type: SECRET_TYPES.GITHUB_TOKEN,
+    expression: GITHUB_TOKEN_PATTERN,
+  },
+  {
+    name: "openAiLikeKey",
+    type: SECRET_TYPES.OPENAI_LIKE_KEY,
+    expression: OPENAI_LIKE_KEY_PATTERN,
+  },
+  {
+    name: "anthropicLikeKey",
+    type: SECRET_TYPES.ANTHROPIC_LIKE_KEY,
+    expression: ANTHROPIC_LIKE_KEY_PATTERN,
+  },
+  {
+    name: "slackToken",
+    type: SECRET_TYPES.SLACK_TOKEN,
+    expression: SLACK_TOKEN_PATTERN,
+  },
+  {
+    name: "privateKeyBlock",
+    type: SECRET_TYPES.PRIVATE_KEY_BLOCK,
+    expression: PRIVATE_KEY_BLOCK_PATTERN,
+  },
+  {
+    name: "passwordAssignment",
+    type: SECRET_TYPES.PASSWORD_ASSIGNMENT,
+    expression: PASSWORD_ASSIGNMENT_PATTERN,
+  },
+  {
+    name: "apiKeyAssignment",
+    type: SECRET_TYPES.API_KEY_ASSIGNMENT,
+    expression: API_KEY_ASSIGNMENT_PATTERN,
+  },
+  {
+    name: "urlCredentials",
+    type: SECRET_TYPES.URL_CREDENTIALS,
+    expression: URL_CREDENTIALS_PATTERN,
+  },
+] as const satisfies readonly SecretPatternDefinition[];
+
+export function matchAwsAccessKeyIds(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[0]);
+}
+
+export function matchGenericBearerTokens(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[1]);
+}
+
+export function matchGitHubTokens(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[2]);
+}
+
+export function matchOpenAiLikeKeys(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[3]);
+}
+
+export function matchAnthropicLikeKeys(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[4]);
+}
+
+export function matchSlackTokens(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[5]);
+}
+
+export function matchPrivateKeyBlocks(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[6]);
+}
+
+export function matchPasswordAssignments(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[7]);
+}
+
+export function matchApiKeyAssignments(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[8]);
+}
+
+export function matchUrlCredentials(value: string): SecretMatch[] {
+  return matchSecretPattern(value, SECRET_PATTERN_DEFINITIONS[9]);
+}
+
+export function matchAllSecretPatterns(value: string): SecretMatch[] {
+  const matches = SECRET_PATTERN_DEFINITIONS.flatMap(definition => matchSecretPattern(value, definition));
+  return matches.sort(compareSecretMatches);
+}
+
+export function formatSecretRedaction(match: Pick<SecretMatch, "type">, sha256Prefix: string): string {
+  return `[REDACTED:${match.type}:${sha256Prefix}]`;
+}
+
+export function matchSecretPattern(value: string, definition: SecretPatternDefinition): SecretMatch[] {
+  const expression = cloneGlobalPattern(definition.expression);
+  const matches = Array.from(value.matchAll(expression));
+  return matches.map(match => toSecretMatch(definition, match));
+}
+
+export function cloneGlobalPattern(expression: RegExp): RegExp {
+  const flags = expression.flags.includes("g") ? expression.flags : `${expression.flags}g`;
+  return new RegExp(expression.source, flags);
+}
+
+export function toSecretMatch(definition: SecretPatternDefinition, match: RegExpMatchArray): SecretMatch {
+  const start = match.index ?? 0;
+  return {
+    type: definition.type,
+    patternName: definition.name,
+    value: match[0],
+    start,
+    end: start + match[0].length,
+  };
+}
+
+export function compareSecretMatches(left: SecretMatch, right: SecretMatch): number {
+  if (left.start !== right.start) return left.start - right.start;
+  if (left.end !== right.end) return right.end - left.end;
+  return left.patternName.localeCompare(right.patternName);
+}

package/src/privacy/truncate.ts

@@ -0,0 +1,69 @@
+import type { LimitsConfig } from "../config/schema.ts";
+import { COMMON_SPAN_ATTRIBUTES } from "../semconv/attributes.ts";
+
+export type ContentLimitKind = "prompt" | "response" | "toolArgument" | "toolResult" | "bashOutput" | "logBody";
+
+export type ContentLimitKey =
+  | "maxPromptChars"
+  | "maxResponseChars"
+  | "maxToolArgumentChars"
+  | "maxToolResultChars"
+  | "maxBashOutputChars"
+  | "maxLogBodyChars";
+
+export type ContentLimits = Pick<LimitsConfig, ContentLimitKey>;
+
+export type TruncationAttributes = Partial<
+  Record<
+    typeof COMMON_SPAN_ATTRIBUTES.OBSERVME_TRUNCATED | typeof COMMON_SPAN_ATTRIBUTES.OBSERVME_ORIGINAL_LENGTH,
+    boolean | number
+  >
+>;
+
+export interface TruncationResult {
+  readonly value: string;
+  readonly truncated: boolean;
+  readonly originalLength?: number;
+  readonly attributes: TruncationAttributes;
+}
+
+export const CONTENT_LIMIT_KEYS = {
+  prompt: "maxPromptChars",
+  response: "maxResponseChars",
+  toolArgument: "maxToolArgumentChars",
+  toolResult: "maxToolResultChars",
+  bashOutput: "maxBashOutputChars",
+  logBody: "maxLogBodyChars",
+} as const satisfies Record<ContentLimitKind, ContentLimitKey>;
+
+export function truncateContent(value: string, kind: ContentLimitKind, limits: ContentLimits): TruncationResult {
+  const limit = limitForContentKind(kind, limits);
+  if (value.length <= limit) return untruncatedContent(value);
+  return truncatedContent(value, limit);
+}
+
+export function limitForContentKind(kind: ContentLimitKind, limits: ContentLimits): number {
+  const limit = limits[CONTENT_LIMIT_KEYS[kind]];
+  if (!Number.isInteger(limit) || limit < 1) throw new Error(`invalid content limit for ${kind}`);
+  return limit;
+}
+
+export function untruncatedContent(value: string): TruncationResult {
+  return {
+    value,
+    truncated: false,
+    attributes: {},
+  };
+}
+
+export function truncatedContent(value: string, limit: number): TruncationResult {
+  return {
+    value: value.slice(0, limit),
+    truncated: true,
+    originalLength: value.length,
+    attributes: {
+      [COMMON_SPAN_ATTRIBUTES.OBSERVME_TRUNCATED]: true,
+      [COMMON_SPAN_ATTRIBUTES.OBSERVME_ORIGINAL_LENGTH]: value.length,
+    },
+  };
+}

package/src/query/grafana-readiness.ts

@@ -0,0 +1,164 @@
+import type { GrafanaDatasourceUidsConfig, ObservMeConfig } from "../config/schema.ts";
+import { hasUnresolvedEnvironmentPlaceholder, normalizeConfiguredGrafanaSecret } from "./grafana-transport.ts";
+
+export type GrafanaQueryDatasourceKey = keyof GrafanaDatasourceUidsConfig;
+export type GrafanaQueryReadinessStatus = "ready" | "disabled" | "not_ready";
+
+export interface GrafanaQueryReadinessIssue {
+  readonly code: string;
+  readonly key: string;
+  readonly message: string;
+}
+
+export interface GrafanaQueryReadinessResult {
+  readonly status: GrafanaQueryReadinessStatus;
+  readonly issues: readonly GrafanaQueryReadinessIssue[];
+}
+
+export function getGrafanaQueryReadiness(
+  config: ObservMeConfig,
+  datasourceKey?: GrafanaQueryDatasourceKey,
+): GrafanaQueryReadinessResult {
+  if (!config.query.enabled) return { status: "disabled", issues: [] };
+
+  const issues = [
+    ...validateGrafanaUrl(config.query.grafana.url),
+    ...validateGrafanaAuth(config),
+    ...validateGrafanaDatasourceUid(config, datasourceKey),
+  ];
+
+  return { status: issues.length === 0 ? "ready" : "not_ready", issues };
+}
+
+export function assertGrafanaQueryReady(config: ObservMeConfig, datasourceKey?: GrafanaQueryDatasourceKey): void {
+  const readiness = getGrafanaQueryReadiness(config, datasourceKey);
+  if (readiness.status !== "not_ready") return;
+
+  throw new Error(formatGrafanaQueryReadiness(readiness));
+}
+
+export function formatGrafanaQueryReadiness(readiness: GrafanaQueryReadinessResult): string {
+  if (readiness.status === "ready") return "Grafana query configuration is ready.";
+  if (readiness.status === "disabled") return "Grafana query integration is disabled (query.enabled=false).";
+
+  return `Grafana query configuration is not ready: ${readiness.issues.map(formatGrafanaQueryReadinessIssue).join(" ")}`;
+}
+
+function validateGrafanaUrl(url: string): GrafanaQueryReadinessIssue[] {
+  const trimmed = url.trim();
+  if (!trimmed) {
+    return [createGrafanaReadinessIssue("missing_grafana_url", "query.grafana.url", "query.grafana.url is not configured.")];
+  }
+
+  if (hasUnresolvedEnvironmentPlaceholder(trimmed)) {
+    return [
+      createGrafanaReadinessIssue(
+        "unresolved_grafana_url",
+        "query.grafana.url",
+        "query.grafana.url is unresolved. Set the referenced environment variable before running Grafana-backed queries.",
+      ),
+    ];
+  }
+
+  return validateGrafanaUrlProtocol(trimmed);
+}
+
+function validateGrafanaUrlProtocol(url: string): GrafanaQueryReadinessIssue[] {
+  try {
+    const parsed = new URL(url);
+    if (parsed.protocol === "http:" || parsed.protocol === "https:") return [];
+  } catch (error) {
+    return [createInvalidGrafanaUrlIssue(readUrlParseFailureKind(error))];
+  }
+
+  return [createInvalidGrafanaUrlIssue()];
+}
+
+function createInvalidGrafanaUrlIssue(failureKind?: string): GrafanaQueryReadinessIssue {
+  const failureDetail = failureKind ? ` URL parser failed with ${failureKind}.` : "";
+  return createGrafanaReadinessIssue(
+    "invalid_grafana_url",
+    "query.grafana.url",
+    `query.grafana.url must be a valid http:// or https:// URL.${failureDetail}`,
+  );
+}
+
+function readUrlParseFailureKind(error: unknown): string {
+  if (error instanceof Error) return error.name || "Error";
+  return typeof error;
+}
+
+function validateGrafanaAuth(config: ObservMeConfig): GrafanaQueryReadinessIssue[] {
+  const token = config.query.grafana.token.trim();
+  const username = config.query.grafana.username.trim();
+  const password = config.query.grafana.password.trim();
+
+  if (normalizeConfiguredGrafanaSecret(token)) return [];
+  if (normalizeConfiguredGrafanaSecret(username) && normalizeConfiguredGrafanaSecret(password)) return [];
+  if (hasUnresolvedEnvironmentPlaceholder(token)) return [createUnresolvedGrafanaTokenIssue()];
+  if (hasUnresolvedEnvironmentPlaceholder(username) || hasUnresolvedEnvironmentPlaceholder(password)) {
+    return [createUnresolvedGrafanaBasicAuthIssue()];
+  }
+  if (username || password) return [createIncompleteGrafanaBasicAuthIssue()];
+  return [createMissingGrafanaAuthIssue()];
+}
+
+function createUnresolvedGrafanaTokenIssue(): GrafanaQueryReadinessIssue {
+  return createGrafanaReadinessIssue(
+    "unresolved_grafana_token",
+    "query.grafana.token",
+    "query.grafana.token is unresolved. Set the referenced environment variable or configure query.grafana.username/password.",
+  );
+}
+
+function createUnresolvedGrafanaBasicAuthIssue(): GrafanaQueryReadinessIssue {
+  return createGrafanaReadinessIssue(
+    "unresolved_grafana_basic_auth",
+    "query.grafana.username/password",
+    "query.grafana.username/password contains an unresolved environment placeholder. Set the referenced environment variables or configure query.grafana.token.",
+  );
+}
+
+function createIncompleteGrafanaBasicAuthIssue(): GrafanaQueryReadinessIssue {
+  return createGrafanaReadinessIssue(
+    "incomplete_grafana_basic_auth",
+    "query.grafana.username/password",
+    "query.grafana.username/password is incomplete. Configure both values or configure query.grafana.token.",
+  );
+}
+
+function createMissingGrafanaAuthIssue(): GrafanaQueryReadinessIssue {
+  return createGrafanaReadinessIssue(
+    "missing_grafana_auth",
+    "query.grafana.token",
+    "query.grafana.token is missing and query.grafana.username/password are not configured.",
+  );
+}
+
+function validateGrafanaDatasourceUid(
+  config: ObservMeConfig,
+  datasourceKey: GrafanaQueryDatasourceKey | undefined,
+): GrafanaQueryReadinessIssue[] {
+  if (!datasourceKey) return [];
+
+  const key = `query.grafana.datasourceUids.${datasourceKey}`;
+  const uid = config.query.grafana.datasourceUids[datasourceKey].trim();
+  if (!uid) return [createGrafanaReadinessIssue("missing_grafana_datasource_uid", key, `${key} is not configured.`)];
+  if (!hasUnresolvedEnvironmentPlaceholder(uid)) return [];
+
+  return [
+    createGrafanaReadinessIssue(
+      "unresolved_grafana_datasource_uid",
+      key,
+      `${key} is unresolved. Set the referenced environment variable before running Grafana-backed queries.`,
+    ),
+  ];
+}
+
+function formatGrafanaQueryReadinessIssue(issue: GrafanaQueryReadinessIssue): string {
+  return issue.message;
+}
+
+function createGrafanaReadinessIssue(code: string, key: string, message: string): GrafanaQueryReadinessIssue {
+  return { code, key, message };
+}