@ponbac/token-ledger 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +25 -2
  2. package/dist/main.js +89 -29
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -493,6 +493,9 @@ Run it on **every machine** where you use a coding agent, and schedule it daily.
493
493
  machine ID and a label taken from the host name. Edit the label freely, but keep
494
494
  the ID: a new ID makes the same history count again as another machine.
495
495
 
496
+ A machine's first upload takes in all the history found on it, back to its earliest
497
+ usage, and says so. Every sync after that covers the last 14 days.
498
+
496
499
  If you map projects in a `token-ledger.json`, pass it with `--config`; scheduled jobs
497
500
  do not run in your project directory. Days follow the system time zone; add
498
501
  `--time-zone Europe/Stockholm` when a scheduler's environment may lack one.
@@ -661,7 +664,27 @@ See [the payload contract](docs/sync-v1.md) for every field.
661
664
 
662
665
  ### Windows and re-syncing
663
666
 
664
- The default window is the last 14 local days. For each provider whose history was
667
+ The default window is the last 14 local days, except for a machine's first upload,
668
+ which reaches back to the earliest usage found on it. Before scanning, and again just
669
+ before sending, `sync` asks the server whether it has stored anything from this
670
+ machine. Only when the server plainly answers that it has not is the whole history
671
+ sent: a long window then has nothing to replace. If the server already lists the
672
+ machine, the sync is an ordinary one, whatever the settings directory holds, so
673
+ restored settings or a lost reply do not lead to a second whole-history upload. If the
674
+ server cannot be asked, `sync` says so, sends the ordinary window, and leaves earlier
675
+ history to `--since`. Two machines sharing one `machine.json` replace each other's
676
+ usage on every sync, and a first upload is no exception.
677
+
678
+ Passing `--since` chooses the window yourself, on a first upload as on any other.
679
+ `sync --dry-run` asks the same question when it has a server and token, and shows the
680
+ payload a real run would send; without them it shows the ordinary window and says why.
681
+ If the server refuses a whole history as too large, `sync` uploads the last 14 days
682
+ instead and says how to send the rest in parts. The first upload is the machine's, not
683
+ each tool's: a tool left out of it with `--provider`, or whose history cannot be read
684
+ then, is not sent in full later by itself. `sync` says so at the time; send that
685
+ tool's earlier history afterwards with `--provider` and `--since`.
686
+
687
+ For each provider whose history was
665
688
  fully read, the server replaces what this machine previously sent for that provider
666
689
  in the window, so repeated syncs never double count, and machines add up. Fully read
667
690
  means every file that could hold usage in the window: older transcripts are
@@ -726,7 +749,7 @@ For manual publication:
726
749
  1. Log in with `npm login` using an account with publish access to `@ponbac/token-ledger`.
727
750
  2. Run `bun run check`. The CLI test packs the package, installs it offline outside
728
751
  the repository with lifecycle scripts disabled, and exercises the installed CLI.
729
- 3. Run `bun run pack` to create `dist/ponbac-token-ledger-0.4.0.tgz` (filename follows version).
752
+ 3. Run `bun run pack` to create `dist/ponbac-token-ledger-0.5.0.tgz` (filename follows version).
730
753
  4. Inspect `npm publish ./dist/npm --dry-run`.
731
754
  5. Publish with `npm publish ./dist/npm --access public`; complete npm's authentication
732
755
  prompt if requested.
package/dist/main.js CHANGED
@@ -35162,6 +35162,7 @@ function planHints() {
35162
35162
 
35163
35163
  // packages/core/src/ledger.ts
35164
35164
  var modifiedSlack = 36 * 60 * 60 * 1000;
35165
+ var historyFloor = Date.UTC(2020, 0, 1);
35165
35166
 
35166
35167
  class InvalidRequest extends exports_Schema.TaggedError()("InvalidRequest", {
35167
35168
  message: exports_Schema.String
@@ -35335,18 +35336,28 @@ class Ledger extends exports_Context.Service()("token-ledger/Ledger") {
35335
35336
  coverage
35336
35337
  });
35337
35338
  });
35338
- const syncPayload = exports_Effect.fn("Ledger.syncPayload")(function* (input, clientInput) {
35339
+ const payloadFrom = exports_Effect.fn("Ledger.payloadFrom")(function* (input, clientInput, history) {
35339
35340
  const request3 = yield* decode5(input);
35340
35341
  const client = yield* exports_Schema.decodeUnknownEffect(SyncClient)(clientInput).pipe(exports_Effect.mapError(() => new InvalidRequest({
35341
35342
  message: "Expected a machine UUID and label and a client version."
35342
35343
  })));
35343
- const window = hourRange(dayWindow(request3.since, request3.until, request3.timeZone));
35344
+ const requested = hourRange(dayWindow(request3.since, request3.until, request3.timeZone));
35345
+ const from = history ? Math.min(historyFloor, requested.start) : requested.start;
35344
35346
  const buckets = sessionBuckets(request3.pricing);
35345
35347
  const hints = planHints();
35346
- const coverage = yield* scan2(request3, window.start, (timestamp2) => timestamp2 >= window.start && timestamp2 < window.end ? timestamp2 : undefined, (record3, attribution) => buckets.add(record3, attribution.shareable ? attribution.project : unattributed), (plan) => hints.add(plan));
35348
+ let earliest = requested.start;
35349
+ const coverage = yield* scan2(request3, from, (timestamp2) => timestamp2 >= from && timestamp2 < requested.end ? timestamp2 : undefined, (record3, attribution, timestamp2) => {
35350
+ earliest = Math.min(earliest, timestamp2);
35351
+ buckets.add(record3, attribution.shareable ? attribution.project : unattributed);
35352
+ }, (plan) => {
35353
+ earliest = Math.min(earliest, plan.timestamp);
35354
+ hints.add(plan);
35355
+ });
35356
+ const since = earliest < requested.start ? exports_DateTime.formatIsoDate(exports_DateTime.setZone(exports_DateTime.makeUnsafe(earliest), request3.timeZone)) : request3.since;
35357
+ const window = since === request3.since ? requested : hourRange(dayWindow(since, request3.until, request3.timeZone));
35347
35358
  const summary = providerCoverage(coverage);
35348
35359
  const replaced = replacedProviders(summary);
35349
- return yield* SyncPayload.makeEffect({
35360
+ const payload = yield* SyncPayload.makeEffect({
35350
35361
  version: 1,
35351
35362
  machine: client.machine,
35352
35363
  clientVersion: client.version,
@@ -35368,8 +35379,13 @@ class Ledger extends exports_Context.Service()("token-ledger/Ledger") {
35368
35379
  }).pipe(exports_Effect.mapError(() => new InvalidRequest({
35369
35380
  message: "Report data must satisfy the sync payload contract."
35370
35381
  })));
35382
+ return { payload, since };
35383
+ });
35384
+ return Ledger.of({
35385
+ report,
35386
+ syncPayload: (request3, client) => exports_Effect.map(payloadFrom(request3, client, false), ({ payload }) => payload),
35387
+ historyPayload: (request3, client) => payloadFrom(request3, client, true)
35371
35388
  });
35372
- return Ledger.of({ report, syncPayload });
35373
35389
  }));
35374
35390
  }
35375
35391
  function acceptsFile(source, file2) {
@@ -42683,7 +42699,7 @@ var runWith2 = (command, config) => {
42683
42699
  // packages/cli/package.json
42684
42700
  var package_default = {
42685
42701
  name: "@ponbac/token-ledger",
42686
- version: "0.4.0",
42702
+ version: "0.5.0",
42687
42703
  private: true,
42688
42704
  type: "module",
42689
42705
  bin: {
@@ -45594,6 +45610,17 @@ class SyncRejected extends exports_Schema.TaggedError()("SyncRejected", {
45594
45610
  }) {
45595
45611
  }
45596
45612
  var Acknowledgement = exports_Schema.Struct({ storedBuckets: exports_Schema.Natural });
45613
+ var MachineList = exports_Schema.Struct({
45614
+ staleAfterDays: exports_Schema.Number,
45615
+ from: exports_Schema.String,
45616
+ to: exports_Schema.String,
45617
+ machines: exports_Schema.Array(exports_Schema.Struct({ machineId: exports_Schema.String.check(exports_Schema.isUUID()) }))
45618
+ });
45619
+ function endpoint(server, path) {
45620
+ const base = new URL(server);
45621
+ base.pathname = base.pathname.endsWith("/") ? base.pathname : `${base.pathname}/`;
45622
+ return new URL(path, base);
45623
+ }
45597
45624
  var VersionRejection = exports_Schema.Struct({ supportedVersions: exports_Schema.Array(exports_Schema.Int) });
45598
45625
  var readResponse = exports_Effect.fn("Sync.readResponse")(function* (response, version) {
45599
45626
  const status = response.status;
@@ -45615,9 +45642,7 @@ var readResponse = exports_Effect.fn("Sync.readResponse")(function* (response, v
45615
45642
  });
45616
45643
  var upload = exports_Effect.fn("Sync.upload")(function* (server, token, payload) {
45617
45644
  const client = (yield* exports_HttpClient.HttpClient).pipe(exports_HttpClient.withScope);
45618
- const base = new URL(server);
45619
- base.pathname = base.pathname.endsWith("/") ? base.pathname : `${base.pathname}/`;
45620
- const url = new URL(`ai-usage/machines/${payload.machine.id}/usage`, base);
45645
+ const url = endpoint(server, `ai-usage/machines/${payload.machine.id}/usage`);
45621
45646
  const request3 = yield* exports_HttpClientRequest.put(url).pipe(exports_HttpClientRequest.bearerToken(token), exports_HttpClientRequest.acceptJson, exports_HttpClientRequest.schemaBodyJson(SyncPayload)(payload), exports_Effect.orDie);
45622
45647
  return yield* client.execute(request3).pipe(exports_Effect.flatMap((response) => readResponse(response, payload.version)), exports_Effect.scoped, exports_Effect.timeout("60 seconds"), exports_Effect.catchTag(["HttpClientError", "TimeoutError"], () => exports_Effect.fail(new SyncUnreachable({ message: `Cannot reach ${url.origin}; try again later.` }))), exports_Effect.retry({
45623
45648
  while: exports_Predicate.isTagged("SyncUnreachable"),
@@ -45625,6 +45650,14 @@ var upload = exports_Effect.fn("Sync.upload")(function* (server, token, payload)
45625
45650
  times: 3
45626
45651
  }));
45627
45652
  });
45653
+ var hasUploaded = exports_Effect.fn("Sync.hasUploaded")(function* (server, token, id2) {
45654
+ const client = (yield* exports_HttpClient.HttpClient).pipe(exports_HttpClient.withScope);
45655
+ const url = endpoint(server, "ai-usage/machines");
45656
+ const unanswered = new SyncUnreachable({
45657
+ message: `Could not ask ${url.origin} whether this machine has uploaded before.`
45658
+ });
45659
+ return yield* client.execute(exports_HttpClientRequest.get(url).pipe(exports_HttpClientRequest.bearerToken(token), exports_HttpClientRequest.acceptJson)).pipe(exports_Effect.filterOrFail((response) => response.status === 200, () => unanswered), exports_Effect.flatMap(exports_HttpClientResponse.schemaBodyJson(MachineList)), exports_Effect.map(({ machines }) => machines.some((machine) => machine.machineId.toLowerCase() === id2.toLowerCase())), exports_Effect.scoped, exports_Effect.timeout("20 seconds"), exports_Effect.provideService(exports_FetchHttpClient.RequestInit, { redirect: "error" }), exports_Effect.mapError(() => unanswered));
45660
+ });
45628
45661
 
45629
45662
  // packages/cli/src/main.ts
45630
45663
  class CliError2 extends exports_Schema.TaggedError()("CliError", { message: exports_Schema.String }) {
@@ -45748,7 +45781,7 @@ var serverFlag = exports_Flag.String("server").pipe(exports_Flag.withSchema(Serv
45748
45781
  var tokenFlag = exports_Flag.Redacted("token").pipe(exports_Flag.withFallbackConfig(exports_Config.Redacted("TOKI_API_TOKEN")), exports_Flag.optional);
45749
45782
  var claudeRetentionDays = 30;
45750
45783
  var sync6 = exports_Command.make("sync", {
45751
- ...usageFlags("13 days before --until", "Exit 2 without uploading for missing/partial sources or unpriced usage"),
45784
+ ...usageFlags("13 days before --until; a machine's first sync goes back to its earliest usage", "Exit 2 without uploading for missing/partial sources or unpriced usage"),
45752
45785
  server: serverFlag.pipe(exports_Flag.withDescription("toki2 URL; otherwise TOKEN_LEDGER_SERVER or sync.json")),
45753
45786
  token: tokenFlag.pipe(exports_Flag.withDescription("toki2 API token; prefer TOKI_API_TOKEN or sync.json")),
45754
45787
  dryRun: exports_Flag.Boolean("dry-run").pipe(exports_Flag.withDefault(false), exports_Flag.withDescription("Print the payload as JSON and upload nothing"))
@@ -45758,42 +45791,69 @@ var sync6 = exports_Command.make("sync", {
45758
45791
  const { request: request3, today } = yield* usageRequest({ ...flags, source: exports_Option.none() }, (today2) => addDays(until ?? today2, -13));
45759
45792
  const machine = yield* loadMachine(directory);
45760
45793
  const ledger = yield* Ledger;
45761
- const payload = yield* ledger.syncPayload(request3, {
45762
- machine,
45763
- version: package_default.version
45764
- });
45765
- for (const entry of payload.coverage) {
45766
- if (entry.status !== "ok")
45767
- yield* exports_Console.error(`${entry.provider}: ${entry.status}; ${entry.files} files, ${entry.filesBeforeWindow ?? 0} older files not opened, ${entry.unreadable} unreadable, ${entry.malformedLines} malformed lines, ${entry.skippedRecords} skipped records.${entry.status === "failed" ? " Not uploaded; the server keeps its stored usage." : ""}`);
45768
- }
45769
- if (request3.sources.some((source) => source.provider === "claude") && request3.since < addDays(today, 1 - claudeRetentionDays))
45794
+ const client = { machine, version: package_default.version };
45795
+ const unsaved = {};
45796
+ const saved = exports_Option.isSome(flags.server) && exports_Option.isSome(flags.token) ? exports_Result.succeed(unsaved) : yield* loadSyncSettings(directory).pipe(exports_Effect.result);
45797
+ const settings = exports_Result.isSuccess(saved) ? saved.success : unsaved;
45798
+ const server = exports_Option.orElse(flags.server, () => exports_Option.fromUndefinedOr(settings.server));
45799
+ const token = exports_Option.orElse(flags.token, () => exports_Option.fromUndefinedOr(settings.token));
45800
+ const unlisted2 = exports_Option.isSome(flags.since) || exports_Option.isNone(server) || exports_Option.isNone(token) ? exports_Option.none() : yield* hasUploaded(server.value, token.value, machine.id).pipe(exports_Effect.map((listed2) => !listed2), exports_Effect.option);
45801
+ const whole = exports_Option.contains(unlisted2, true);
45802
+ const reachable = exports_Option.isSome(server) && exports_Option.isSome(token);
45803
+ if (exports_Option.isNone(unlisted2) && exports_Option.isNone(flags.since) && (reachable || flags.dryRun))
45804
+ yield* exports_Console.error(`${reachable ? "Could not ask the server whether this machine has uploaded before" : "No server and API token to ask whether this machine has uploaded before"}, so the window is ${request3.since} through ${request3.until}. If this is its first upload, send its earlier history with --since.`);
45805
+ const first = whole ? yield* ledger.historyPayload(request3, client) : { payload: yield* ledger.syncPayload(request3, client), since: request3.since };
45806
+ if (whole)
45807
+ yield* exports_Console.error(`First upload from this machine: taking in all the history found, from ${first.since}. Later syncs cover the last 14 days.${flags.provider === "all" ? "" : ` Only ${flags.provider} is included; the other tools' earlier history is not sent later by itself, so send it with --provider and --since.`}`);
45808
+ const examine = exports_Effect.fn("CLI.examine")(function* (payload) {
45809
+ for (const entry of payload.coverage) {
45810
+ if (entry.status !== "ok")
45811
+ yield* exports_Console.error(`${entry.provider}: ${entry.status}; ${entry.files} files, ${entry.filesBeforeWindow ?? 0} older files not opened, ${entry.unreadable} unreadable, ${entry.malformedLines} malformed lines, ${entry.skippedRecords} skipped records.${entry.status === "failed" ? " Not uploaded; the server keeps its stored usage." : ""}`);
45812
+ if (whole && entry.status === "failed")
45813
+ yield* exports_Console.error(`${entry.provider}: once its history can be read, send the earlier part with token-ledger sync --provider ${entry.provider} --since <first day>.`);
45814
+ }
45815
+ return payload.coverage.some((entry) => entry.status !== "ok") || payload.buckets.some((bucket) => bucket.unpricedRecords > 0);
45816
+ });
45817
+ const incomplete = yield* examine(first.payload);
45818
+ if (!whole && request3.sources.some((source) => source.provider === "claude") && request3.since < addDays(today, 1 - claudeRetentionDays))
45770
45819
  yield* exports_Console.error(`Warning: the window starts more than ${claudeRetentionDays} days ago. Claude Code deletes older transcripts by default, and the server replaces the whole window, so pruned history would lower its totals.`);
45771
- const incomplete = payload.coverage.some((entry) => entry.status !== "ok") || payload.buckets.some((bucket) => bucket.unpricedRecords > 0);
45772
45820
  if (flags.dryRun) {
45773
- yield* exports_Console.log(JSON.stringify(payload, null, 2));
45821
+ yield* exports_Console.log(JSON.stringify(first.payload, null, 2));
45774
45822
  if (flags.strict && incomplete)
45775
45823
  process.exitCode = 2;
45776
45824
  return;
45777
45825
  }
45778
- if (flags.strict && incomplete) {
45826
+ const refused = exports_Effect.gen(function* () {
45779
45827
  yield* exports_Console.error("Not syncing: coverage is incomplete or usage is unpriced (--strict).");
45780
45828
  process.exitCode = 2;
45781
- return;
45782
- }
45783
- const settings = exports_Option.isSome(flags.server) && exports_Option.isSome(flags.token) ? {} : yield* loadSyncSettings(directory);
45784
- const server = exports_Option.orElse(flags.server, () => exports_Option.fromUndefinedOr(settings.server));
45785
- const token = exports_Option.orElse(flags.token, () => exports_Option.fromUndefinedOr(settings.token));
45829
+ });
45830
+ if (flags.strict && incomplete)
45831
+ return yield* refused;
45832
+ if (exports_Result.isFailure(saved))
45833
+ return yield* exports_Effect.fail(saved.failure);
45786
45834
  if (exports_Option.isNone(server) || exports_Option.isNone(token))
45787
45835
  return yield* new CliError2({
45788
45836
  message: `Missing ${exports_Option.isNone(server) ? "server" : "API token"}: pass --server/--token, set TOKEN_LEDGER_SERVER/TOKI_API_TOKEN, or add it to ${directory}/sync.json.`
45789
45837
  });
45790
- const outcome = yield* upload(server.value, token.value, payload).pipe(exports_Effect.result);
45838
+ const send = (payload) => upload(server.value, token.value, payload).pipe(exports_Effect.result);
45839
+ const usual = exports_Effect.fn("CLI.usual")(function* (reason) {
45840
+ yield* exports_Console.error(`${reason} Uploading ${request3.since} through ${request3.until} instead; send earlier history in parts with --since and --until.`);
45841
+ const payload = yield* ledger.syncPayload(request3, client);
45842
+ return (yield* examine(payload)) && flags.strict ? exports_Option.none() : exports_Option.some({ sent: { payload, since: request3.since }, outcome: yield* send(payload) });
45843
+ });
45844
+ const still = !whole || !(yield* hasUploaded(server.value, token.value, machine.id).pipe(exports_Effect.orElseSucceed(() => true)));
45845
+ let result6 = still ? exports_Option.some({ sent: first, outcome: yield* send(first.payload) }) : yield* usual("This machine has uploaded since it was asked about.");
45846
+ if (whole && exports_Option.isSome(result6) && exports_Result.isFailure(result6.value.outcome) && exports_Match.value(result6.value.outcome.failure).pipe(exports_Match.tag("SyncRejected", () => still), exports_Match.orElse(() => false)))
45847
+ result6 = yield* usual(result6.value.outcome.failure.message);
45848
+ if (exports_Option.isNone(result6))
45849
+ return yield* refused;
45850
+ const { sent, outcome } = result6.value;
45791
45851
  if (exports_Result.isFailure(outcome)) {
45792
45852
  yield* exports_Console.error(outcome.failure.message);
45793
45853
  process.exitCode = exports_Match.value(outcome.failure).pipe(exports_Match.tag("SyncUnreachable", () => 3), exports_Match.tag("SyncUnauthorized", () => 4), exports_Match.tag("SyncUnsupportedVersion", () => 5), exports_Match.tag("SyncRejected", () => 6), exports_Match.exhaustive);
45794
45854
  return;
45795
45855
  }
45796
- yield* exports_Console.log(syncSummary(payload, request3, outcome.success));
45856
+ yield* exports_Console.log(syncSummary(sent.payload, { ...request3, since: sent.since }, outcome.success));
45797
45857
  })).pipe(exports_Command.withDescription("Upload hourly usage buckets from this machine to toki2; the server replaces the window"));
45798
45858
  var interactive = (prompt) => exports_Prompt.run(prompt).pipe(exports_Effect.mapError(() => new CliError2({
45799
45859
  message: "Nothing was saved. Without a terminal, pass --server and set TOKI_API_TOKEN."
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ponbac/token-ledger",
3
- "version": "0.4.0",
3
+ "version": "0.5.0",
4
4
  "description": "Local AI token usage and API-equivalent cost reports by project",
5
5
  "license": "MIT",
6
6
  "engines": {