@rawdash/connector-twilio 0.0.1

package/dist/index.js

@@ -0,0 +1,574 @@
+// ../../connector-shared/dist/index.js
+var HTTP_CLIENT_VERSION = "0.0.0";
+var DEFAULT_USER_AGENT = `rawdash-connector/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
+function connectorUserAgent(connectorId) {
+  return `rawdash-connector-${connectorId}/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
+}
+function sanitizeAllowedUrl(options) {
+  const { url, host, pathname, protocol = "https:" } = options;
+  if (url === null) {
+    return null;
+  }
+  try {
+    const u = new URL(url);
+    if (u.protocol !== protocol || u.host !== host || u.pathname !== pathname) {
+      return null;
+    }
+    return u.toString();
+  } catch {
+    return null;
+  }
+}
+function parseEpoch(value, unit) {
+  if (value === null || value === void 0) {
+    return null;
+  }
+  if (unit === "iso") {
+    if (typeof value !== "string") {
+      return null;
+    }
+    const ms = new Date(value).getTime();
+    return Number.isFinite(ms) ? ms : null;
+  }
+  if (typeof value === "string" && value.trim() === "") {
+    return null;
+  }
+  const n = typeof value === "number" ? value : Number(value);
+  if (!Number.isFinite(n)) {
+    return null;
+  }
+  const result = unit === "s" ? n * 1e3 : n;
+  return Number.isFinite(result) ? result : null;
+}
+
+// src/twilio.ts
+import {
+  BaseConnector,
+  defineConfigFields,
+  defineConnectorDoc,
+  defineResources,
+  makeChunkedCursorGuard,
+  metricSample,
+  paginateChunked,
+  schemasFromResources,
+  selectActivePhases
+} from "@rawdash/core";
+import { z } from "zod";
+var TWILIO_API_HOST = "api.twilio.com";
+var TWILIO_API_BASE = `https://${TWILIO_API_HOST}`;
+var API_VERSION = "2010-04-01";
+var PAGE_SIZE = 1e3;
+var MS_PER_DAY = 864e5;
+var DEFAULT_LOOKBACK_DAYS = 30;
+var INCREMENTAL_LOOKBACK_DAYS = 2;
+var configFields = defineConfigFields(
+  z.object({
+    accountSid: z.string().min(1).meta({
+      label: "Account SID",
+      description: "Twilio Account SID (starts with AC). Found on the Twilio Console dashboard. Used as the Basic auth username and in every request path.",
+      placeholder: "AC..."
+    }),
+    authToken: z.object({ $secret: z.string().min(1) }).meta({
+      label: "Auth token",
+      description: "Twilio Auth token for the account, used as the Basic auth password. Prefer a standard Auth token; an API key secret also works when paired with its SID as the Account SID.",
+      placeholder: "TWILIO_AUTH_TOKEN",
+      secret: true
+    }),
+    resources: z.array(
+      z.enum([
+        "twilio_message",
+        "twilio_call",
+        "twilio_usage_count",
+        "twilio_usage_price"
+      ])
+    ).nonempty().optional().meta({
+      label: "Resources",
+      description: "Which Twilio resources to sync. Omit to sync all of them. The two usage metrics share one upstream call to the daily Usage Records report."
+    }),
+    lookbackDays: z.number().int().positive().max(365).optional().meta({
+      label: "Backfill window (days)",
+      description: "How many days of usage history to fetch on a full sync. Defaults to 30. Message and call backfill is bounded by the same window.",
+      placeholder: "30"
+    })
+  })
+);
+var doc = defineConnectorDoc({
+  displayName: "Twilio",
+  category: "engineering",
+  brandColor: "#F22F46",
+  tagline: "Track SMS and voice volume, delivery and error rates, and per-category spend from the Twilio REST API.",
+  vendor: {
+    name: "Twilio",
+    domain: "twilio.com",
+    apiDocs: "https://www.twilio.com/docs/usage/api",
+    website: "https://twilio.com"
+  },
+  auth: {
+    summary: "Authenticates over HTTP Basic auth using the Twilio Account SID as the username and the Auth token as the password. Read access to messages, calls, and usage records is sufficient.",
+    setup: [
+      "Open the Twilio Console dashboard and copy your Account SID (starts with AC).",
+      "Copy the Auth token shown next to it, or create a standard API key (Console -> Account -> API keys & tokens) and use its SID and secret.",
+      "Store the token as a secret (e.g. TWILIO_AUTH_TOKEN).",
+      'Reference it from config as `authToken: secret("TWILIO_AUTH_TOKEN")` alongside `accountSid: "AC..."`.'
+    ]
+  },
+  rateLimit: "Twilio returns 429 with a Retry-After header when the per-account concurrency budget is exceeded; the shared HTTP client honors it. List endpoints paginate via a relative next_page_uri with a configurable PageSize (capped at 1000 here).",
+  limitations: [
+    "Monetary amounts (message/call price, usage price) are reported by Twilio as negative-signed decimal strings; the connector stores their absolute value as a positive number.",
+    "Message and call events are bounded by the backfill window; very high-volume accounts should sync the usage metrics rather than per-message events for spend and volume trends.",
+    "Usage is read from the daily Usage Records report (1-day granularity); sub-daily usage is not exposed."
+  ]
+});
+var PHASE_ORDER = ["messages", "calls", "usage"];
+var isTwilioSyncCursor = makeChunkedCursorGuard(PHASE_ORDER);
+var RESOURCES_BY_PHASE = {
+  messages: ["twilio_message"],
+  calls: ["twilio_call"],
+  usage: ["twilio_usage_count", "twilio_usage_price"]
+};
+var ENDPOINT_PATH = {
+  messages: `/${API_VERSION}/Accounts/{sid}/Messages.json`,
+  calls: `/${API_VERSION}/Accounts/{sid}/Calls.json`,
+  usage: `/${API_VERSION}/Accounts/{sid}/Usage/Records/Daily.json`
+};
+var messageSchema = z.object({
+  sid: z.string().min(1),
+  status: z.string().nullish(),
+  error_code: z.number().int().nullish(),
+  direction: z.string().nullish(),
+  price: z.string().nullish(),
+  price_unit: z.string().nullish(),
+  date_sent: z.string().nullish(),
+  date_created: z.string().nullish(),
+  from: z.string().nullish(),
+  to: z.string().nullish(),
+  num_segments: z.string().nullish(),
+  num_media: z.string().nullish(),
+  messaging_service_sid: z.string().nullish()
+});
+var callSchema = z.object({
+  sid: z.string().min(1),
+  status: z.string().nullish(),
+  direction: z.string().nullish(),
+  duration: z.string().nullish(),
+  price: z.string().nullish(),
+  price_unit: z.string().nullish(),
+  start_time: z.string().nullish(),
+  end_time: z.string().nullish(),
+  date_created: z.string().nullish(),
+  from: z.string().nullish(),
+  to: z.string().nullish()
+});
+var usageRecordSchema = z.object({
+  category: z.string().min(1),
+  description: z.string().nullish(),
+  count: z.string().nullish(),
+  count_unit: z.string().nullish(),
+  usage: z.string().nullish(),
+  usage_unit: z.string().nullish(),
+  price: z.string().nullish(),
+  price_unit: z.string().nullish(),
+  start_date: z.string().nullish(),
+  end_date: z.string().nullish()
+});
+var messagesResponseSchema = z.object({
+  messages: z.array(messageSchema),
+  next_page_uri: z.string().nullish()
+});
+var callsResponseSchema = z.object({
+  calls: z.array(callSchema),
+  next_page_uri: z.string().nullish()
+});
+var usageResponseSchema = z.object({
+  usage_records: z.array(usageRecordSchema),
+  next_page_uri: z.string().nullish()
+});
+var USAGE_DIMENSIONS = [
+  {
+    name: "category",
+    description: "Twilio usage category (e.g. sms, sms-inbound, sms-outbound, calls, calls-inbound, calls-outbound, verify, whatsapp)."
+  },
+  {
+    name: "description",
+    description: "Human-readable label for the category, or null."
+  }
+];
+var twilioResources = defineResources({
+  twilio_message: {
+    shape: "event",
+    filterable: [],
+    description: "SMS / MMS message attempts with status, error code, direction, and price, timestamped at the time the message was sent.",
+    endpoint: "GET /2010-04-01/Accounts/{AccountSid}/Messages.json",
+    notes: "start_ts is date_sent when present, falling back to date_created. Messages whose timestamp cannot be parsed are skipped.",
+    fields: [
+      { name: "sid", description: "Twilio message SID." },
+      {
+        name: "status",
+        description: "Delivery status (queued, sending, sent, delivered, undelivered, failed, received, ...)."
+      },
+      {
+        name: "errorCode",
+        description: "Twilio error code if the message failed, else null."
+      },
+      {
+        name: "direction",
+        description: "Message direction (inbound, outbound-api, outbound-call, outbound-reply)."
+      },
+      {
+        name: "price",
+        description: "Absolute price charged for the message in priceUnit, or null if not yet priced."
+      },
+      {
+        name: "priceUnit",
+        description: "ISO currency code for price, or null."
+      },
+      { name: "from", description: "Sender address or number." },
+      { name: "to", description: "Recipient address or number." },
+      {
+        name: "numSegments",
+        description: "Number of message segments billed."
+      },
+      { name: "numMedia", description: "Number of media attachments." },
+      {
+        name: "messagingServiceSid",
+        description: "Messaging Service SID the message was sent through, or null."
+      }
+    ],
+    responses: { messages: messagesResponseSchema }
+  },
+  twilio_call: {
+    shape: "event",
+    filterable: [],
+    description: "Voice call attempts with status, direction, duration, and price, timestamped at the call start time.",
+    endpoint: "GET /2010-04-01/Accounts/{AccountSid}/Calls.json",
+    notes: "start_ts is start_time when present, falling back to date_created. Calls whose timestamp cannot be parsed are skipped.",
+    fields: [
+      { name: "sid", description: "Twilio call SID." },
+      {
+        name: "status",
+        description: "Call status (queued, ringing, in-progress, completed, busy, failed, no-answer, canceled)."
+      },
+      {
+        name: "direction",
+        description: "Call direction (inbound, outbound-api, outbound-dial)."
+      },
+      {
+        name: "duration",
+        description: "Call duration in seconds.",
+        unit: "seconds"
+      },
+      {
+        name: "price",
+        description: "Absolute price charged for the call in priceUnit, or null if not yet priced."
+      },
+      {
+        name: "priceUnit",
+        description: "ISO currency code for price, or null."
+      },
+      { name: "from", description: "Caller number." },
+      { name: "to", description: "Callee number." }
+    ],
+    responses: { calls: callsResponseSchema }
+  },
+  twilio_usage_count: {
+    shape: "metric",
+    description: "Daily usage count per Twilio billing category, from the daily Usage Records report.",
+    endpoint: "GET /2010-04-01/Accounts/{AccountSid}/Usage/Records/Daily.json",
+    unit: "count",
+    granularity: "daily",
+    dimensions: [...USAGE_DIMENSIONS],
+    measures: [
+      {
+        name: "usage",
+        description: "Raw usage amount in the category usage_unit (may differ from count, e.g. seconds for voice)."
+      }
+    ],
+    notes: "Sample value is the Usage Record count. Written from the same usage call as twilio_usage_price.",
+    responses: { usage_records: usageResponseSchema }
+  },
+  twilio_usage_price: {
+    shape: "metric",
+    description: "Daily spend per Twilio billing category, from the daily Usage Records report.",
+    endpoint: "GET /2010-04-01/Accounts/{AccountSid}/Usage/Records/Daily.json",
+    unit: "currency",
+    granularity: "daily",
+    dimensions: [...USAGE_DIMENSIONS],
+    notes: "Sample value is the absolute Usage Record price in priceUnit. Written alongside twilio_usage_count from one usage call."
+  }
+});
+var twilioCredentials = {
+  authToken: {
+    description: "Twilio Auth token (Basic auth password)",
+    auth: "required"
+  }
+};
+var id = "twilio";
+function absNumber(value) {
+  if (value === null || value === void 0 || value.trim() === "") {
+    return null;
+  }
+  const n = Number.parseFloat(value);
+  return Number.isFinite(n) ? Math.abs(n) : null;
+}
+function intCount(value) {
+  if (value === null || value === void 0 || value.trim() === "") {
+    return 0;
+  }
+  const n = Number.parseFloat(value);
+  return Number.isFinite(n) ? n : 0;
+}
+function messageStartTs(message) {
+  return parseEpoch(message.date_sent ?? null, "iso") ?? parseEpoch(message.date_created ?? null, "iso");
+}
+function callStartTs(call) {
+  return parseEpoch(call.start_time ?? null, "iso") ?? parseEpoch(call.date_created ?? null, "iso");
+}
+function buildMessageEvents(messages) {
+  const events = [];
+  for (const m of messages) {
+    const ts = messageStartTs(m);
+    if (ts === null) {
+      continue;
+    }
+    events.push({
+      name: "twilio_message",
+      start_ts: ts,
+      end_ts: null,
+      attributes: {
+        sid: m.sid,
+        status: m.status ?? null,
+        errorCode: m.error_code ?? null,
+        direction: m.direction ?? null,
+        price: absNumber(m.price),
+        priceUnit: m.price_unit ?? null,
+        from: m.from ?? null,
+        to: m.to ?? null,
+        numSegments: intCount(m.num_segments),
+        numMedia: intCount(m.num_media),
+        messagingServiceSid: m.messaging_service_sid ?? null
+      }
+    });
+  }
+  return events;
+}
+function buildCallEvents(calls) {
+  const events = [];
+  for (const c of calls) {
+    const ts = callStartTs(c);
+    if (ts === null) {
+      continue;
+    }
+    events.push({
+      name: "twilio_call",
+      start_ts: ts,
+      end_ts: null,
+      attributes: {
+        sid: c.sid,
+        status: c.status ?? null,
+        direction: c.direction ?? null,
+        duration: intCount(c.duration),
+        price: absNumber(c.price),
+        priceUnit: c.price_unit ?? null,
+        from: c.from ?? null,
+        to: c.to ?? null
+      }
+    });
+  }
+  return events;
+}
+function buildUsageSamples(records) {
+  const counts = [];
+  const prices = [];
+  for (const r of records) {
+    const ts = parseEpoch(r.start_date ?? null, "iso");
+    if (ts === null) {
+      continue;
+    }
+    const dims = {
+      category: r.category,
+      description: r.description ?? null
+    };
+    counts.push(
+      metricSample(twilioResources, "twilio_usage_count", {
+        ts,
+        value: intCount(r.count),
+        attributes: { ...dims, usage: intCount(r.usage) }
+      })
+    );
+    prices.push(
+      metricSample(twilioResources, "twilio_usage_price", {
+        ts,
+        value: absNumber(r.price) ?? 0,
+        attributes: { ...dims }
+      })
+    );
+  }
+  return { counts, prices };
+}
+function resourceToPhase(resource) {
+  for (const phase of PHASE_ORDER) {
+    if (RESOURCES_BY_PHASE[phase].includes(resource)) {
+      return phase;
+    }
+  }
+  throw new Error(`twilio: unmapped resource ${resource}`);
+}
+function toDate(ms) {
+  return new Date(ms).toISOString().slice(0, 10);
+}
+var TwilioConnector = class _TwilioConnector extends BaseConnector {
+  static id = id;
+  static resources = twilioResources;
+  static schemas = schemasFromResources(twilioResources);
+  static create(input, ctx) {
+    const parsed = configFields.parse(input);
+    return new _TwilioConnector(
+      {
+        accountSid: parsed.accountSid,
+        resources: parsed.resources,
+        lookbackDays: parsed.lookbackDays
+      },
+      { authToken: parsed.authToken },
+      ctx
+    );
+  }
+  id = id;
+  credentials = twilioCredentials;
+  buildHeaders() {
+    const basic = btoa(
+      `${this.settings.accountSid}:${String(this.creds.authToken)}`
+    );
+    return {
+      Authorization: `Basic ${basic}`,
+      Accept: "application/json",
+      "User-Agent": connectorUserAgent(this.id)
+    };
+  }
+  fetch(url, resource, signal) {
+    return this.get(url, {
+      resource,
+      headers: this.buildHeaders(),
+      signal
+    });
+  }
+  phasePath(phase) {
+    return ENDPOINT_PATH[phase].replace("{sid}", this.settings.accountSid);
+  }
+  buildInitialUrl(phase, options, lookbackDays, now) {
+    const url = new URL(`${TWILIO_API_BASE}${this.phasePath(phase)}`);
+    url.searchParams.set("PageSize", String(PAGE_SIZE));
+    if (phase === "usage") {
+      const days = options.mode === "latest" ? INCREMENTAL_LOOKBACK_DAYS : lookbackDays;
+      const sinceMs2 = options.since ? parseEpoch(options.since, "iso") : null;
+      const startMs = sinceMs2 !== null ? Math.min(sinceMs2, now - INCREMENTAL_LOOKBACK_DAYS * MS_PER_DAY) : now - days * MS_PER_DAY;
+      url.searchParams.set("StartDate", toDate(startMs));
+      url.searchParams.set("EndDate", toDate(now));
+      return url.toString();
+    }
+    const sinceMs = options.since ? parseEpoch(options.since, "iso") : now - lookbackDays * MS_PER_DAY;
+    if (sinceMs !== null) {
+      const field = phase === "messages" ? "DateSent>" : "StartTime>";
+      url.searchParams.set(field, toDate(sinceMs));
+    }
+    return url.toString();
+  }
+  nextUrl(phase, nextPageUri) {
+    if (!nextPageUri) {
+      return null;
+    }
+    return sanitizeAllowedUrl({
+      url: `${TWILIO_API_BASE}${nextPageUri}`,
+      host: TWILIO_API_HOST,
+      pathname: this.phasePath(phase)
+    });
+  }
+  async writePhase(storage, phase, items) {
+    switch (phase) {
+      case "messages":
+        await storage.events(buildMessageEvents(items), {
+          names: ["twilio_message"]
+        });
+        return;
+      case "calls":
+        await storage.events(buildCallEvents(items), {
+          names: ["twilio_call"]
+        });
+        return;
+      case "usage": {
+        const { counts, prices } = buildUsageSamples(
+          items
+        );
+        await storage.metrics(counts, { names: ["twilio_usage_count"] });
+        await storage.metrics(prices, { names: ["twilio_usage_price"] });
+        return;
+      }
+    }
+  }
+  parsePage(phase, body) {
+    switch (phase) {
+      case "messages": {
+        const parsed = messagesResponseSchema.parse(body);
+        return {
+          items: parsed.messages,
+          nextPageUri: parsed.next_page_uri ?? null
+        };
+      }
+      case "calls": {
+        const parsed = callsResponseSchema.parse(body);
+        return {
+          items: parsed.calls,
+          nextPageUri: parsed.next_page_uri ?? null
+        };
+      }
+      case "usage": {
+        const parsed = usageResponseSchema.parse(body);
+        return {
+          items: parsed.usage_records,
+          nextPageUri: parsed.next_page_uri ?? null
+        };
+      }
+    }
+  }
+  async sync(options, storage, signal) {
+    const cursor = isTwilioSyncCursor(options.cursor) ? options.cursor : void 0;
+    const lookbackDays = this.settings.lookbackDays ?? DEFAULT_LOOKBACK_DAYS;
+    const now = Date.now();
+    const phases = selectActivePhases(
+      resourceToPhase,
+      PHASE_ORDER,
+      this.settings.resources
+    );
+    return paginateChunked({
+      phases,
+      cursor,
+      signal,
+      logger: this.logger,
+      fetchPage: async (phase, page, sig) => {
+        const url = page ?? this.buildInitialUrl(phase, options, lookbackDays, now);
+        const res = await this.fetch(url, phase, sig);
+        const { items, nextPageUri } = this.parsePage(phase, res.body);
+        return { items, next: this.nextUrl(phase, nextPageUri) };
+      },
+      writeBatch: async (phase, items) => {
+        await this.writePhase(storage, phase, items);
+      }
+    });
+  }
+};
+
+// src/index.ts
+var index_default = TwilioConnector;
+export {
+  TwilioConnector,
+  buildCallEvents,
+  buildMessageEvents,
+  buildUsageSamples,
+  callStartTs,
+  configFields,
+  index_default as default,
+  doc,
+  id,
+  messageStartTs,
+  twilioResources as resources
+};
+//# sourceMappingURL=index.js.map