@oneuptime/common 12.0.24 → 12.0.26

package/Server/Services/TelemetrySourceMapService.ts

@@ -0,0 +1,307 @@
+import CreateBy from "../Types/Database/CreateBy";
+import { OnCreate } from "../Types/Database/Hooks";
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/TelemetrySourceMap";
+import BadDataException from "../../Types/Exception/BadDataException";
+import ObjectID from "../../Types/ObjectID";
+import SortOrder from "../../Types/BaseDatabase/SortOrder";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+import QueryHelper from "../Types/Database/QueryHelper";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import SourceMapResolver, {
+  MAX_SOURCE_MAP_SIZE_IN_BYTES,
+  SourceMapBundle,
+} from "../Utils/Telemetry/SourceMapResolver";
+import {
+  MinifiedStackFrame,
+  ResolvedStackFrame,
+  ResolveStackTraceResult,
+} from "../../Types/Telemetry/SourceMap";
+
+export { MAX_SOURCE_MAP_SIZE_IN_BYTES };
+
+/*
+ * How long uploaded maps are kept. Old releases age out automatically —
+ * a map is only useful while exceptions from its release are still within
+ * telemetry retention, and 90 days comfortably exceeds the default.
+ */
+export const SOURCE_MAP_RETENTION_DAYS: number = 90;
+
+/*
+ * How many maps one (service, release) pair may hold. Enforced by the
+ * upload endpoint (existing bundles + new bundles must fit) and used as
+ * the resolver's read limit, so nothing stored is ever silently ignored.
+ * A build rarely emits more than a few dozen chunks with maps.
+ */
+export const MAX_SOURCE_MAPS_PER_RELEASE: number = 100;
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+    /*
+     * Age-based retention: the HardDelete:HardDeleteOlderItemsInDatabase
+     * worker sweeps this daily (not gated on billing, so self-hosted
+     * deployments get cleanup too).
+     */
+    this.hardDeleteItemsOlderThanInDays("createdAt", SOURCE_MAP_RETENTION_DAYS);
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    const content: string | undefined = createBy.data.content;
+
+    if (!content) {
+      throw new BadDataException("Source map content is required.");
+    }
+
+    if (Buffer.byteLength(content, "utf8") > MAX_SOURCE_MAP_SIZE_IN_BYTES) {
+      throw new BadDataException(
+        `Source map is too large. The maximum size is ${
+          MAX_SOURCE_MAP_SIZE_IN_BYTES / (1024 * 1024)
+        } MB.`,
+      );
+    }
+
+    SourceMapResolver.validateSourceMapContent(content);
+
+    if (!createBy.data.bundlePath || !createBy.data.bundlePath.trim()) {
+      throw new BadDataException("Bundle path is required.");
+    }
+
+    if (!createBy.data.serviceVersion || !createBy.data.serviceVersion.trim()) {
+      throw new BadDataException("Service version is required.");
+    }
+
+    createBy.data.sizeInBytes = Buffer.byteLength(content, "utf8");
+
+    return { createBy, carryForward: null };
+  }
+
+  /**
+   * Bundle paths currently stored for a (project, service, release) tuple.
+   * Used by the upload endpoint to enforce the per-release ceiling without
+   * pulling map content.
+   */
+  @CaptureSpan()
+  public async getStoredBundlePathsForRelease(data: {
+    projectId: ObjectID;
+    serviceId: ObjectID;
+    serviceVersion: string;
+  }): Promise<Array<string>> {
+    const rows: Array<Model> = await this.findBy({
+      query: {
+        projectId: data.projectId,
+        serviceId: data.serviceId,
+        serviceVersion: data.serviceVersion,
+      },
+      select: {
+        _id: true,
+        bundlePath: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    return rows
+      .map((row: Model) => {
+        return row.bundlePath || "";
+      })
+      .filter((bundlePath: string) => {
+        return bundlePath.length > 0;
+      });
+  }
+
+  /**
+   * Store a source map for (project, service, release, bundle), replacing
+   * any previous upload for the same bundle so CI re-runs converge on one
+   * row. Used by the authenticated upload endpoint, hence isRoot.
+   */
+  @CaptureSpan()
+  public async replaceSourceMap(data: {
+    projectId: ObjectID;
+    serviceId: ObjectID;
+    serviceVersion: string;
+    bundlePath: string;
+    content: string;
+  }): Promise<Model> {
+    /*
+     * Hard delete (not soft): a replaced map has no restore story, and
+     * soft-deleted rows would hold megabytes until a billing-gated sweep
+     * that self-hosted deployments never run.
+     */
+    await this.hardDeleteBy({
+      query: {
+        projectId: data.projectId,
+        serviceId: data.serviceId,
+        serviceVersion: data.serviceVersion,
+        bundlePath: data.bundlePath,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    const sourceMap: Model = new Model();
+    sourceMap.projectId = data.projectId;
+    sourceMap.serviceId = data.serviceId;
+    sourceMap.serviceVersion = data.serviceVersion;
+    sourceMap.bundlePath = data.bundlePath;
+    sourceMap.content = data.content;
+
+    return this.create({
+      data: sourceMap,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
+  /**
+   * Resolve parsed exception stack frames for a service + release against
+   * the maps uploaded for that pair. Reads with isRoot — callers must have
+   * already authorized the (projectId, serviceId) pair.
+   */
+  @CaptureSpan()
+  public async resolveFramesForService(data: {
+    projectId: ObjectID;
+    serviceId: ObjectID;
+    serviceVersion: string;
+    frames: Array<MinifiedStackFrame>;
+  }): Promise<ResolveStackTraceResult> {
+    if (!data.serviceVersion || data.frames.length === 0) {
+      return {
+        frames: data.frames.map((frame: MinifiedStackFrame) => {
+          return { ...frame, resolved: false };
+        }),
+        resolvedCount: 0,
+        sourceMapCount: 0,
+      };
+    }
+
+    /*
+     * Two-step fetch: list bundle paths first (cheap), then pull content
+     * only for bundles that at least one frame can match. Maps can be tens
+     * of megabytes each and a release may store up to
+     * MAX_SOURCE_MAPS_PER_RELEASE of them — loading them all for every
+     * exception view would be a memory hazard for no benefit.
+     */
+    const sourceMapRows: Array<Model> = await this.findBy({
+      query: {
+        projectId: data.projectId,
+        serviceId: data.serviceId,
+        serviceVersion: data.serviceVersion,
+      },
+      select: {
+        _id: true,
+        bundlePath: true,
+        createdAt: true,
+      },
+      sort: {
+        createdAt: SortOrder.Descending,
+      },
+      limit: MAX_SOURCE_MAPS_PER_RELEASE,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    /*
+     * Newest-first sort + first-wins dedupe: if a racing double upload left
+     * two rows for one bundle, the newer one is used.
+     */
+    const dedupedRows: Array<Model> = [];
+    const seenBundlePaths: Set<string> = new Set();
+
+    for (const row of sourceMapRows) {
+      if (!row.bundlePath) {
+        continue;
+      }
+
+      const normalizedPath: string = SourceMapResolver.normalizePath(
+        row.bundlePath,
+      );
+
+      if (seenBundlePaths.has(normalizedPath)) {
+        continue;
+      }
+
+      seenBundlePaths.add(normalizedPath);
+      dedupedRows.push(row);
+    }
+
+    const matchedRows: Array<Model> = dedupedRows.filter((row: Model) => {
+      return data.frames.some((frame: MinifiedStackFrame) => {
+        return (
+          SourceMapResolver.getMatchScore(
+            frame.fileName,
+            row.bundlePath || "",
+          ) > 0
+        );
+      });
+    });
+
+    const bundles: Array<SourceMapBundle> = [];
+
+    if (matchedRows.length > 0) {
+      const contentRows: Array<Model> = await this.findBy({
+        query: {
+          /*
+           * Tenant scoping repeated here even though ids are already
+           * tenant-scoped — defense in depth for a root query.
+           */
+          projectId: data.projectId,
+          _id: QueryHelper.any(
+            matchedRows.map((row: Model) => {
+              return new ObjectID(row.id!.toString());
+            }),
+          ),
+        },
+        select: {
+          _id: true,
+          bundlePath: true,
+          content: true,
+        },
+        limit: MAX_SOURCE_MAPS_PER_RELEASE,
+        skip: 0,
+        props: {
+          isRoot: true,
+        },
+      });
+
+      for (const row of contentRows) {
+        if (row.bundlePath && row.content) {
+          bundles.push({
+            bundlePath: row.bundlePath,
+            content: row.content,
+          });
+        }
+      }
+    }
+
+    const resolvedFrames: Array<ResolvedStackFrame> =
+      SourceMapResolver.resolveFrames(data.frames, bundles);
+
+    return {
+      frames: resolvedFrames,
+      resolvedCount: resolvedFrames.filter((frame: ResolvedStackFrame) => {
+        return frame.resolved;
+      }).length,
+      /*
+       * Stored (deduped) maps for the release — not just the ones a frame
+       * matched — so "maps exist but none matched your bundles" is
+       * distinguishable from "no maps uploaded".
+       */
+      sourceMapCount: dedupedRows.length,
+    };
+  }
+}
+
+export default new Service();

package/Server/Services/ThreatIntelFeedService.ts

@@ -0,0 +1,222 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/ThreatIntelFeed";
+import CreateBy from "../Types/Database/CreateBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import BadDataException from "../../Types/Exception/BadDataException";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+import ObjectID from "../../Types/ObjectID";
+import {
+  THREAT_INTEL_MINIMUM_CONFIDENCE_MAX,
+  THREAT_INTEL_MINIMUM_CONFIDENCE_MIN,
+} from "../../Types/SecurityEvent/ThreatIntelConstants";
+import TaxiiClient from "../Utils/SecurityEvent/ThreatIntel/TaxiiClient";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  /*
+   * Validate at save time so a feed that stores is a feed the poller can
+   * use — a malformed URL or collection id surfaces to the person
+   * configuring it, not as a cron-side lastError later. (The
+   * GoogleSecOpsConnectionService discipline.)
+   */
+  private validateFeed(data: {
+    apiRootUrl?: string | undefined;
+    collectionId?: string | undefined;
+    pollIntervalInMinutes?: number | undefined;
+    minimumConfidence?: number | undefined;
+  }): void {
+    if (data.apiRootUrl !== undefined) {
+      TaxiiClient.validateApiRootUrl(data.apiRootUrl);
+    }
+
+    if (data.collectionId !== undefined) {
+      TaxiiClient.validateCollectionId(data.collectionId);
+    }
+
+    if (data.pollIntervalInMinutes !== undefined) {
+      if (
+        !Number.isInteger(data.pollIntervalInMinutes) ||
+        data.pollIntervalInMinutes < 1 ||
+        data.pollIntervalInMinutes > 1440
+      ) {
+        throw new BadDataException(
+          "Poll interval must be a whole number of minutes between 1 and 1440.",
+        );
+      }
+    }
+
+    if (data.minimumConfidence !== undefined) {
+      if (
+        !Number.isInteger(data.minimumConfidence) ||
+        data.minimumConfidence < THREAT_INTEL_MINIMUM_CONFIDENCE_MIN ||
+        data.minimumConfidence > THREAT_INTEL_MINIMUM_CONFIDENCE_MAX
+      ) {
+        throw new BadDataException(
+          `Minimum confidence must be a whole number between ${THREAT_INTEL_MINIMUM_CONFIDENCE_MIN} and ${THREAT_INTEL_MINIMUM_CONFIDENCE_MAX}.`,
+        );
+      }
+    }
+  }
+
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    if (!createBy.data.apiRootUrl) {
+      throw new BadDataException("TAXII API root URL is required.");
+    }
+
+    if (!createBy.data.collectionId) {
+      throw new BadDataException("Collection ID is required.");
+    }
+
+    if (createBy.data.apiToken && createBy.data.basicAuthPassword) {
+      throw new BadDataException(
+        "Configure either an API token or basic-auth credentials, not both.",
+      );
+    }
+
+    this.validateFeed({
+      apiRootUrl: createBy.data.apiRootUrl,
+      collectionId: createBy.data.collectionId,
+      pollIntervalInMinutes: createBy.data.pollIntervalInMinutes,
+      minimumConfidence: createBy.data.minimumConfidence,
+    });
+
+    return { createBy, carryForward: null };
+  }
+
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    this.validateFeed({
+      apiRootUrl: updateBy.data.apiRootUrl as string | undefined,
+      collectionId: updateBy.data.collectionId as string | undefined,
+      pollIntervalInMinutes: updateBy.data.pollIntervalInMinutes as
+        | number
+        | undefined,
+      minimumConfidence: updateBy.data.minimumConfidence as number | undefined,
+    });
+
+    await this.validateCredentialExclusivity(updateBy);
+
+    /*
+     * Repointing a feed invalidates its poll position: the added_after
+     * cursor (and any saved page token) is a position in ONE collection's
+     * date_added timeline. Carrying it to a different API root or
+     * collection would silently skip the new collection's entire history.
+     * Flagged here, reset in onUpdateSuccess — the cursor columns have
+     * update: [] access control, so writing them into updateBy.data would
+     * be rejected for non-root callers; the root-props write after the
+     * update bypasses the column ACL the same legitimate way the poller's
+     * own bookkeeping does.
+     */
+    const repointed: boolean =
+      updateBy.data.apiRootUrl !== undefined ||
+      updateBy.data.collectionId !== undefined;
+
+    return { updateBy, carryForward: { repointed } };
+  }
+
+  protected override async onUpdateSuccess(
+    onUpdate: OnUpdate<Model>,
+    updatedItemIds: Array<ObjectID>,
+  ): Promise<OnUpdate<Model>> {
+    if (
+      (onUpdate.carryForward as { repointed?: boolean } | null)?.repointed ===
+      true
+    ) {
+      for (const updatedItemId of updatedItemIds) {
+        /*
+         * Cannot recurse: this inner update carries neither apiRootUrl
+         * nor collectionId, so its own onBeforeUpdate flags nothing.
+         */
+        await this.updateOneById({
+          id: updatedItemId,
+          data: {
+            cursor: null as unknown as string,
+            nextPageToken: null as unknown as string,
+            lastPolledAt: null as unknown as Date,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+    }
+
+    return onUpdate;
+  }
+
+  /*
+   * The create-time either/or rule, enforced on updates too — including
+   * against what the row ALREADY stores, since the API updates secrets
+   * one at a time. TaxiiClient silently prefers the token over basic
+   * auth, so a row holding both would poll with whichever secret the
+   * user was NOT trying to use.
+   */
+  private async validateCredentialExclusivity(
+    updateBy: UpdateBy<Model>,
+  ): Promise<void> {
+    const settingToken: boolean = Boolean(updateBy.data.apiToken);
+    const settingBasicPassword: boolean = Boolean(
+      updateBy.data.basicAuthPassword,
+    );
+
+    if (settingToken && settingBasicPassword) {
+      throw new BadDataException(
+        "Configure either an API token or basic-auth credentials, not both.",
+      );
+    }
+
+    if (!settingToken && !settingBasicPassword) {
+      return;
+    }
+
+    /*
+     * Setting one kind while the update leaves the other kind's stored
+     * value untouched (undefined = not in the update; null = explicit
+     * clear, which is fine) — check the matched rows.
+     */
+    const otherKindUntouched: boolean = settingToken
+      ? updateBy.data.basicAuthPassword === undefined
+      : updateBy.data.apiToken === undefined;
+
+    if (!otherKindUntouched) {
+      return;
+    }
+
+    const matchedFeeds: Array<Model> = await this.findBy({
+      query: updateBy.query,
+      select: {
+        _id: true,
+        apiToken: true,
+        basicAuthPassword: true,
+      },
+      skip: 0,
+      limit: LIMIT_MAX,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const feed of matchedFeeds) {
+      const storesOtherKind: boolean = settingToken
+        ? Boolean(feed.basicAuthPassword)
+        : Boolean(feed.apiToken);
+
+      if (storesOtherKind) {
+        throw new BadDataException(
+          settingToken
+            ? "This feed uses basic auth. Clear the basic-auth password in the same update (or use the Update Credentials action, which switches automatically) before setting an API token."
+            : "This feed uses an API token. Clear the API token in the same update (or use the Update Credentials action, which switches automatically) before setting a basic-auth password.",
+        );
+      }
+    }
+  }
+}
+
+export default new Service();