@oneuptime/common 12.0.7 → 12.0.8

package/Models/DatabaseModels/TelemetryEntity.ts

@@ -15,6 +15,7 @@ import IconProp from "../../Types/Icon/IconProp";
 import { JSONObject } from "../../Types/JSON";
 import ObjectID from "../../Types/ObjectID";
 import Permission from "../../Types/Permission";
+import EntitySource from "../../Types/Telemetry/EntitySource";
 import EntityType from "../../Types/Telemetry/EntityType";
 import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 import DatabaseBaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
@@ -187,6 +188,50 @@ export default class TelemetryEntity extends DatabaseBaseModel {
   })
   public displayName?: string = undefined;
 
+  /*
+   * Immutable after create (`update: []`): the value decides whether
+   * PruneStaleEntities may reap the row, so letting it be edited would let a
+   * user flip a manual CI into a TTL-pruned one — or quietly make a
+   * discovered row immortal. Re-create the row instead.
+   */
+  @ColumnAccessControl({ create: CREATE_PERMS, read: READ_PERMS, update: [] })
+  @TableColumn({
+    type: TableColumnType.ShortText,
+    required: true,
+    title: "Source",
+    description:
+      "How this row came to exist: discovered from telemetry, mirrored from a OneUptime inventory table, or created manually by a user. Determines whether stale-entity pruning applies.",
+    example: "discovered",
+  })
+  @Index()
+  @Column({
+    type: ColumnType.ShortText,
+    length: ColumnLength.ShortText,
+    nullable: false,
+    default: EntitySource.Discovered,
+  })
+  public source?: EntitySource = undefined;
+
+  @ColumnAccessControl({
+    create: CREATE_PERMS,
+    read: READ_PERMS,
+    update: UPDATE_PERMS,
+  })
+  @TableColumn({
+    type: TableColumnType.LongText,
+    required: false,
+    title: "Description",
+    description:
+      "Free-text description. Primarily for manually created entities, where there are no telemetry attributes to explain what the thing is.",
+    example: "Stripe payments API - vendor managed, no telemetry.",
+  })
+  @Column({
+    type: ColumnType.LongText,
+    length: ColumnLength.LongText,
+    nullable: true,
+  })
+  public description?: string = undefined;
+
   @ColumnAccessControl({
     create: CREATE_PERMS,
     read: READ_PERMS,

package/Models/DatabaseModels/TelemetryEntityRelationship.ts

@@ -15,6 +15,7 @@ import IconProp from "../../Types/Icon/IconProp";
 import ObjectID from "../../Types/ObjectID";
 import Permission from "../../Types/Permission";
 import EntityRelationshipType from "../../Types/Telemetry/EntityRelationshipType";
+import EntitySource from "../../Types/Telemetry/EntitySource";
 import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 import DatabaseBaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
 
@@ -173,6 +174,31 @@ export default class TelemetryEntityRelationship extends DatabaseBaseModel {
   })
   public relationshipType?: EntityRelationshipType = undefined;
 
+  /*
+   * Immutable after create, for the same reason as TelemetryEntity.source:
+   * it is the predicate the edge prune keys on. An edge drawn by hand — the
+   * only way to connect a manual CI to anything — has nothing re-bumping its
+   * `lastSeenAt`, so without this it would be reaped by the 30-day sweep
+   * while the entities at both ends survived.
+   */
+  @ColumnAccessControl({ create: CREATE_PERMS, read: READ_PERMS, update: [] })
+  @TableColumn({
+    type: TableColumnType.ShortText,
+    required: true,
+    title: "Source",
+    description:
+      "Whether this edge was derived from telemetry or drawn manually by a user. Determines whether stale-edge pruning applies.",
+    example: "discovered",
+  })
+  @Index()
+  @Column({
+    type: ColumnType.ShortText,
+    length: ColumnLength.ShortText,
+    nullable: false,
+    default: EntitySource.Discovered,
+  })
+  public source?: EntitySource = undefined;
+
   @ColumnAccessControl({
     create: CREATE_PERMS,
     read: READ_PERMS,

package/Models/DatabaseModels/WorkflowLog.ts

@@ -348,6 +348,44 @@ export default class WorkflowLog extends BaseModel {
   })
   public resumeData?: JSONObject = undefined;
 
+  /*
+   * What each step of the run did, as data rather than as lines in `logs`:
+   * the component, its resolved arguments, what it returned, which out port it
+   * took and how long it took. Readable by anyone who can read the log itself,
+   * because it is the same information in a shape that can be rendered as a
+   * list of steps instead of a wall of text.
+   *
+   * Sensitive arguments and return values are redacted before they get here,
+   * by the same rules that redact them in `logs`.
+   */
+  @ColumnAccessControl({
+    create: [],
+    read: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.Viewer,
+      Permission.WorkflowAdmin,
+      Permission.WorkflowMember,
+      Permission.WorkflowViewer,
+      Permission.ReadWorkflowLog,
+    ],
+    update: [],
+  })
+  @TableColumn({
+    required: false,
+    isDefaultValueColumn: false,
+    type: TableColumnType.JSON,
+    title: "Step Trace",
+    description:
+      "Structured per-step record of this run: arguments, return values, the port taken and timing.",
+  })
+  @Column({
+    type: ColumnType.JSON,
+    nullable: true,
+  })
+  public stepTrace?: JSONObject = undefined;
+
   @ColumnAccessControl({
     create: [],
     read: [],

package/Server/API/UserAPI.ts

@@ -53,10 +53,25 @@ export default class UserAPI extends BaseAPI<User, UserServiceType> {
      * the gate; the read below runs as root because a master admin is not a
      * member of the projects being listed and so has no tenant permissions to
      * read them with.
+     *
+     * The gate accepts the instance-wide master API key as well as a
+     * master-admin session, because the caller asking "which projects is this
+     * person in?" is typically a script rather than somebody with a browser
+     * open. That is a widening, so it is worth being explicit about why it is
+     * the right one HERE and not on the routes below: this endpoint only reads,
+     * and it reads nothing the key does not already reach through
+     * POST /team-member/get-list, which any master-key caller can call today
+     * (untenanted, so cross-project) to assemble the same answer one membership
+     * at a time. The key buys convenience and correct paging, not new reach.
+     *
+     * The writes below stay on the session-only middleware. A leaked static key
+     * that can list memberships is a disclosure of what it could already
+     * disclose; one that can remove a user from a project, set their password or
+     * mail them a reset link is a headless account takeover.
      */
     this.router.post(
       `${new this.entityType().getCrudApiPath()?.toString()}/:userId/projects`,
-      MasterAdminAuthorization.isAuthorizedMasterAdminMiddleware,
+      MasterAdminAuthorization.isAuthorizedMasterAdminOrMasterApiKeyMiddleware,
       async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
         try {
           const userId: ObjectID = UserAPI.getUserIdFromParams(req);

package/Server/Infrastructure/Postgres/SchemaMigrations/1786551733814-MigrationName.ts

@@ -0,0 +1,41 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1786551733814 implements MigrationInterface {
+  public name: string = "MigrationName1786551733814";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "TelemetryEntity" ADD "source" character varying(100) NOT NULL DEFAULT 'discovered'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "TelemetryEntity" ADD "description" character varying(500)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "TelemetryEntityRelationship" ADD "source" character varying(100) NOT NULL DEFAULT 'discovered'`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_8106b1d6443d982b12e3318318" ON "TelemetryEntity" ("source") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_26ea726e2ee8fa622348581ac1" ON "TelemetryEntityRelationship" ("source") `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_26ea726e2ee8fa622348581ac1"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_8106b1d6443d982b12e3318318"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "TelemetryEntityRelationship" DROP COLUMN "source"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "TelemetryEntity" DROP COLUMN "description"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "TelemetryEntity" DROP COLUMN "source"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786559879134-AddWorkflowLogStepTrace.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddWorkflowLogStepTrace1786559879134
+  implements MigrationInterface
+{
+  public name: string = "AddWorkflowLogStepTrace1786559879134";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "WorkflowLog" ADD "stepTrace" jsonb`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "WorkflowLog" DROP COLUMN "stepTrace"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -515,6 +515,8 @@ import { AddDataSourceTable1786303472848 } from "./1786303472848-AddDataSourceTa
 import { MigrationName1786446545142 } from "./1786446545142-MigrationName";
 import { AddMonitorDependency1786449497966 } from "./1786449497966-AddMonitorDependency";
 import { AddDeletedProjectTable1786461136170 } from "./1786461136170-AddDeletedProjectTable";
+import { MigrationName1786551733814 } from "./1786551733814-MigrationName";
+import { AddWorkflowLogStepTrace1786559879134 } from "./1786559879134-AddWorkflowLogStepTrace";
 
 export default [
   InitialMigration,
@@ -1034,4 +1036,6 @@ export default [
   MigrationName1786446545142,
   AddMonitorDependency1786449497966,
   AddDeletedProjectTable1786461136170,
+  MigrationName1786551733814,
+  AddWorkflowLogStepTrace1786559879134,
 ];

package/Server/Middleware/MasterAdminAuthorization.ts

@@ -59,13 +59,18 @@ export default class MasterAdminAuthorization {
    * Same as isAuthorizedMasterAdminMiddleware, but ALSO accepts the instance-wide
    * master API key (Admin Dashboard → Settings → API Key) supplied in the
    * `apikey` header. The master key has root/master-admin access, so this lets
-   * automated callers reach the master-admin OneUptime Health endpoints with the
-   * key instead of a logged-in master-admin session.
+   * automated callers reach master-admin endpoints with the key instead of a
+   * logged-in master-admin session.
    *
-   * Deliberately scoped to the read-only health / diagnostics routes only. It is
-   * NOT used on higher-risk master-admin actions (e.g. the read/write query
-   * console or broadcast email), which stay on the JWT-only middleware above so
-   * that a leaked static key cannot trigger them headlessly.
+   * Deliberately scoped to READ-ONLY routes: the OneUptime Health / diagnostics
+   * endpoints, and the cross-tenant read of one user's projects
+   * (POST /user/:userId/projects). It is NOT used on master-admin routes that
+   * change anything — the read/write query console, broadcast email, removing a
+   * user from a project, setting a password — which stay on the JWT-only
+   * middleware above so that a leaked static key cannot trigger them headlessly.
+   *
+   * Read-only is the line to hold when adding a route here. A key that can read
+   * discloses; a key that can write takes over.
    */
   public static async isAuthorizedMasterAdminOrMasterApiKeyMiddleware(
     req: ExpressRequest,

package/Server/Services/TelemetryEntityRelationshipService.ts

@@ -5,6 +5,7 @@ import OneUptimeDate from "../../Types/Date";
 import logger from "../Utils/Logger";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { EntityRelationshipEdge } from "../../Utils/Telemetry/EntityRelationship";
+import EntitySource from "../../Types/Telemetry/EntitySource";
 import { reconcileByNaturalKey } from "../Utils/Telemetry/EntityRegistry";
 import QueryDeepPartialEntity from "../../Types/Database/PartialEntity";
 
@@ -62,6 +63,8 @@ export class TelemetryEntityRelationshipService extends DatabaseService<Model> {
         model.fromEntityKey = edge.fromEntityKey;
         model.toEntityKey = edge.toEntityKey;
         model.relationshipType = edge.relationshipType;
+        // Explicit: this is what makes the edge eligible for stale pruning.
+        model.source = EntitySource.Discovered;
         model.firstSeenAt = now;
         model.lastSeenAt = now;
         if (edge.metrics) {

package/Server/Services/TelemetryEntityService.ts

@@ -8,8 +8,19 @@ import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import ColumnLength from "../../Types/Database/ColumnLength";
 import QueryDeepPartialEntity from "../../Types/Database/PartialEntity";
+import CreateBy from "../Types/Database/CreateBy";
+import { OnCreate } from "../Types/Database/Hooks";
+import BadDataException from "../../Types/Exception/BadDataException";
+import Dictionary from "../../Types/Dictionary";
 import { JSONObject } from "../../Types/JSON";
+import EntitySource from "../../Types/Telemetry/EntitySource";
 import EntityType from "../../Types/Telemetry/EntityType";
+import { MANUAL_ENTITY_TYPES } from "../../Types/Telemetry/EntityTypeGroups";
+import {
+  canonicalizeEntityValue,
+  computeEntityKey,
+  MANUAL_ENTITY_IDENTITY_ATTRIBUTE,
+} from "../../Utils/Telemetry/EntityKey";
 import logger from "../Utils/Logger";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { ExtractedEntity } from "../Utils/Telemetry/TelemetryEntity";
@@ -25,6 +36,91 @@ export class TelemetryEntityService extends DatabaseService<Model> {
     super(Model);
   }
 
+  /**
+   * Completes a manually created CI.
+   *
+   * Every machine writer — the ingest reconciler and the inventory mirror —
+   * computes `entityKey` itself and passes an explicit `source`, because
+   * both derive identity from something the user never sees (a canonicalized
+   * semconv attribute set, or the owning row's id). A create arriving
+   * WITHOUT a key is therefore a user create by construction, and this hook
+   * is what turns the two fields a human can reasonably supply (type and
+   * name) into the identity the registry needs.
+   *
+   * Deriving the key here rather than in the dashboard keeps it enforced for
+   * API clients too, and keeps the one identity rule — canonicalize, then
+   * hash — in the module that owns it.
+   */
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    const data: Model = createBy.data;
+
+    if (!data.entityKey) {
+      data.source = EntitySource.Manual;
+    }
+
+    if (data.source !== EntitySource.Manual) {
+      return { createBy, carryForward: null };
+    }
+
+    if (!data.projectId) {
+      throw new BadDataException("Project ID is required.");
+    }
+
+    if (!data.entityType) {
+      throw new BadDataException("Entity Type is required.");
+    }
+
+    /*
+     * Confined to the manual-only vocabulary. A hand-made row of an
+     * observable type would key off the manual identity attribute and so
+     * could never converge with the row ingest derives for the same thing —
+     * it would just sit beside it forever. See MANUAL_ENTITY_TYPES.
+     */
+    if (!MANUAL_ENTITY_TYPES.has(data.entityType)) {
+      throw new BadDataException(
+        `Entities of type ${data.entityType} are discovered automatically and cannot be created manually. Manually creatable types are: ${Array.from(
+          MANUAL_ENTITY_TYPES,
+        ).join(", ")}.`,
+      );
+    }
+
+    const displayName: string = (data.displayName || "").trim();
+
+    if (!displayName) {
+      throw new BadDataException(
+        "Name is required for a manually created entity.",
+      );
+    }
+
+    data.displayName = displayName.substring(0, ColumnLength.ShortText);
+
+    const identifyingAttributes: Dictionary<string> = {
+      [MANUAL_ENTITY_IDENTITY_ATTRIBUTE]: canonicalizeEntityValue(displayName),
+    };
+    data.identifyingAttributes = identifyingAttributes as JSONObject;
+
+    data.entityKey = computeEntityKey({
+      projectId: data.projectId.toString(),
+      entityType: data.entityType,
+      identifyingAttributes,
+    });
+
+    /*
+     * Stamped so the explorer can sort manual CIs alongside discovered ones
+     * on the same columns. `lastSeenAt` never moves again for these rows —
+     * it means "created", not "observed" — which is exactly why the prune
+     * sweep excludes them.
+     */
+    const now: Date = OneUptimeDate.getCurrentDate();
+    data.firstSeenAt = data.firstSeenAt || now;
+    data.lastSeenAt = data.lastSeenAt || now;
+
+    return { createBy, carryForward: null };
+  }
+
   /**
    * Forward-only registry reconciliation: upsert a catalog row per
    * discovered entity, bump `lastSeenAt`, merge descriptive attributes
@@ -169,7 +265,18 @@ export class TelemetryEntityService extends DatabaseService<Model> {
         if (count === undefined) {
           count = (
             await this.countBy({
-              query: { projectId, entityType: entity.entityType },
+              query: {
+                projectId,
+                entityType: entity.entityType,
+                /*
+                 * The budget exists to bound rows minted by ingest, so it
+                 * counts only those. Manual CIs and inventory mirrors are
+                 * bounded by their own creators and must not consume it —
+                 * otherwise registering enough devices could stop new
+                 * telemetry entities being recorded at all.
+                 */
+                source: EntitySource.Discovered,
+              },
               props: { isRoot: true },
             })
           ).toNumber();
@@ -198,6 +305,12 @@ export class TelemetryEntityService extends DatabaseService<Model> {
         model.projectId = projectId;
         model.entityType = entity.entityType;
         model.entityKey = entity.entityKey;
+        /*
+         * Explicit rather than leaning on the column default: this is the
+         * flag that makes the row eligible for stale-entity pruning, and a
+         * silently-defaulted value is a poor way to carry that.
+         */
+        model.source = EntitySource.Discovered;
         model.identifyingAttributes = entity.identifyingAttributes;
         if (
           entity.descriptiveAttributes &&

package/Server/Types/Workflow/Components/API/Utils.ts

@@ -4,7 +4,7 @@ import HTTPResponse from "../../../../../Types/API/HTTPResponse";
 import URL from "../../../../../Types/API/URL";
 import BadDataException from "../../../../../Types/Exception/BadDataException";
 import Exception from "../../../../../Types/Exception/Exception";
-import { JSONObject } from "../../../../../Types/JSON";
+import { JSONObject, JSONValue } from "../../../../../Types/JSON";
 import JSONFunctions from "../../../../../Types/JSONFunctions";
 import ComponentMetadata, {
   Port,
@@ -82,6 +82,49 @@ export class ApiComponentUtils {
       );
     }
 
+    /*
+     * Headers reach here from a key/value grid that can emit numbers and
+     * booleans, from a JSON document someone typed, or from a {{...}}
+     * substitution — and every consumer below blind-casts the result to
+     * Dictionary<string> before spreading it into the outgoing request. An
+     * array or a bare scalar spreads into per-index junk headers rather than
+     * failing, so check the shape and flatten the values to strings here.
+     */
+    if (args["request-headers"]) {
+      const headers: JSONValue = args["request-headers"] as JSONValue;
+
+      if (
+        typeof headers !== "object" ||
+        headers === null ||
+        Array.isArray(headers)
+      ) {
+        throw options.onError(
+          new BadDataException(
+            "Request headers must be a JSON object of header names and values.",
+          ),
+        );
+      }
+
+      const stringifiedHeaders: JSONObject = {};
+
+      for (const headerName of Object.keys(headers as JSONObject)) {
+        const headerValue: JSONValue = (headers as JSONObject)[
+          headerName
+        ] as JSONValue;
+
+        if (headerValue === null || headerValue === undefined) {
+          continue;
+        }
+
+        stringifiedHeaders[headerName] =
+          typeof headerValue === "object"
+            ? JSON.stringify(headerValue)
+            : String(headerValue);
+      }
+
+      args["request-headers"] = stringifiedHeaders;
+    }
+
     if (!args["url"]) {
       throw options.onError(new BadDataException("URL not found"));
     }

package/Server/Types/Workflow/TriggerCode.ts

@@ -20,6 +20,18 @@ export interface ExecuteWorkflowType {
    * should leave this undefined.
    */
   callChain?: Array<string>;
+  /**
+   * When set, the run executes only this one component instead of starting at
+   * the trigger and following the graph. Backs the builder's "run just this
+   * step" button.
+   *
+   * Deliberately routed through the queue like any other run, so it inherits
+   * every gate a run has: the workflow must be enabled, the subscription must
+   * be paid, the plan's run limit applies, a WorkflowLog is written for the
+   * audit trail, and the component executes on a worker rather than inside the
+   * API process.
+   */
+  runOnlyComponentId?: string | undefined;
 }
 
 export interface InitProps {

package/Server/Types/Workflow/Workflow.ts

@@ -19,4 +19,9 @@ export interface RunProps {
    * depth across workflow boundaries.
    */
   callChain?: Array<string>;
+  /**
+   * Execute only this component (by its user-facing id) instead of walking the
+   * graph from the trigger. See ExecuteWorkflowType.runOnlyComponentId.
+   */
+  runOnlyComponentId?: string | undefined;
 }