@atolis-hq/wake 0.3.45 → 0.3.46

package/dist/src/bootstrap/version.js

@@ -108,4 +108,4 @@ export function resolveWakeVersion(options = {}) {
         return `g${headHash.slice(0, 7)}`;
     return '0.1.0-dev';
 }
-export const wakeVersion = "g8f84f8d";
+export const wakeVersion = "g88c27c1";

package/dist/src/integrations/application/poll-service.js

@@ -20,6 +20,7 @@ export class PollService {
             await this.journal.append(stream, existing.length, [{ ...draft, stream }]);
             appended += 1;
         }
+        await this.instance.source.markPollPersisted?.();
         return appended;
     }
 }

package/dist/src/integrations/github/infrastructure/client-reads.js

@@ -2,10 +2,10 @@
 import { PullRequestState } from '../../../activities/index.js';
 import { GitHubListState } from '../contracts/vocabulary.js';
 import { fetchPaginatedWithEtag, fetchWithEtag } from './etag-cache.js';
-export function listIssues(octokit, cache, owner, repo, maxResults) {
+export function listIssues(octokit, cache, owner, repo, maxResults, since) {
     return fetchPaginatedWithEtag({
         cache,
-        key: `issues:${owner}/${repo}`,
+        key: `issues:${owner}/${repo}:since:${since ?? 'bootstrap'}`,
         maxResults,
         pages: (headers) => octokit.paginate.iterator(octokit.rest.issues.listForRepo, {
             owner,
@@ -14,6 +14,7 @@ export function listIssues(octokit, cache, owner, repo, maxResults) {
             sort: 'updated',
             direction: 'desc',
             per_page: Math.min(maxResults, 100),
+            ...(since === undefined ? {} : { since }),
             ...(headers === undefined ? {} : { headers }),
         }),
     }).then((items) => items.map(normalizeIssue));
@@ -41,6 +42,8 @@ export async function listPullRequests(octokit, cache, owner, repo, maxResults)
             owner,
             repo,
             state: GitHubListState.All,
+            sort: 'updated',
+            direction: 'desc',
             per_page: Math.min(maxResults, 100),
             ...(headers === undefined ? {} : { headers }),
         }),
@@ -208,15 +211,16 @@ function normalizePullRequestState(pullRequest) {
             : PullRequestState.Open,
     };
 }
-export async function listIssueComments(octokit, cache, owner, repo, issueNumber, pageSize) {
+export async function listIssueComments(octokit, cache, owner, repo, issueNumber, options) {
     const comments = await fetchPaginatedWithEtag({
         cache,
-        key: `issue-comments:${owner}/${repo}#${issueNumber}`,
+        key: `issue-comments:${owner}/${repo}#${issueNumber}:since:${options.since ?? 'bootstrap'}`,
         pages: (headers) => octokit.paginate.iterator(octokit.rest.issues.listComments, {
             owner,
             repo,
             issue_number: issueNumber,
-            per_page: Math.min(pageSize, 100),
+            per_page: Math.min(options.pageSize, 100),
+            ...(options.since === undefined ? {} : { since: options.since }),
             ...(headers === undefined ? {} : { headers }),
         }),
     });

package/dist/src/integrations/github/infrastructure/client.js

@@ -34,9 +34,12 @@ export function createGitHubClient(token) {
     const cache = createEtagCache();
     return {
         authenticatedLogin: async () => (await octokit.rest.users.getAuthenticated()).data.login,
-        listIssues: (owner, repo, maxResults) => listIssues(octokit, cache, owner, repo, maxResults),
+        listIssues: (owner, repo, maxResults, since) => listIssues(octokit, cache, owner, repo, maxResults, since),
         listPullRequests: (owner, repo, maxResults) => listPullRequests(octokit, cache, owner, repo, maxResults),
-        listIssueComments: (owner, repo, issueNumber, pageSize) => listIssueComments(octokit, cache, owner, repo, issueNumber, pageSize),
+        listIssueComments: (owner, repo, issueNumber, pageSize, since) => listIssueComments(octokit, cache, owner, repo, issueNumber, {
+            pageSize,
+            ...(since === undefined ? {} : { since }),
+        }),
         listReviewComments: (owner, repo, pullNumber, pageSize) => listReviewComments(octokit, cache, owner, repo, pullNumber, pageSize),
         collaboratorPermission: async (owner, repo, login) => {
             const { data } = await octokit.rest.repos.getCollaboratorPermissionLevel({

package/dist/src/integrations/github/infrastructure/comment-source.js

@@ -0,0 +1,86 @@
+import { ReviewerAuthorizationSource, } from '../../../activities/index.js';
+import { issueCommentObservation } from './issue-source.js';
+import { mergeBatches, reportPartialPollFailure } from './poll-watermark.js';
+import { githubReviewObservation } from './review-source.js';
+export async function reviewCommentEventsFor(context, pullRequests) {
+    if (context.client.listReviewComments === undefined)
+        return { drafts: [], succeeded: true };
+    const items = await Promise.all(pullRequests.map(async (pullRequest) => {
+        try {
+            const comments = await context.client.listReviewComments(context.owner, context.repo, pullRequest.number, context.config.polling.commentPageSize);
+            return {
+                succeeded: true,
+                drafts: (await Promise.all(comments.map((comment) => issueCommentEventsForComment(context, pullRequest, comment)))).flat(),
+            };
+        }
+        catch {
+            reportPartialPollFailure(context.repository, `pull-request review comments #${pullRequest.number}`);
+            return { drafts: [], succeeded: false };
+        }
+    }));
+    return mergeBatches(items);
+}
+export async function reviewEventsFor(context, pullRequests) {
+    const items = await Promise.all(pullRequests.map(async (pullRequest) => {
+        try {
+            const reviews = await context.client.listReviews(context.owner, context.repo, pullRequest.number, context.config.polling.commentPageSize);
+            return {
+                succeeded: true,
+                drafts: reviews.flatMap((review) => githubReviewObservation({
+                    repository: context.repository,
+                    pullRequest,
+                    review,
+                    authorizedReviewers: [],
+                })),
+            };
+        }
+        catch {
+            reportPartialPollFailure(context.repository, `pull-request reviews #${pullRequest.number}`);
+            return { drafts: [], succeeded: false };
+        }
+    }));
+    return mergeBatches(items);
+}
+export async function issueCommentEventsFor(context, issues, since) {
+    const items = await Promise.all(issues.map(async (issue) => {
+        try {
+            const comments = await context.client.listIssueComments(context.owner, context.repo, issue.number, context.config.polling.commentPageSize, since);
+            return {
+                succeeded: true,
+                drafts: (await Promise.all(comments.map((comment) => issueCommentEventsForComment(context, issue, comment)))).flat(),
+            };
+        }
+        catch {
+            reportPartialPollFailure(context.repository, `issue comments #${issue.number}`);
+            return { drafts: [], succeeded: false };
+        }
+    }));
+    return mergeBatches(items);
+}
+async function issueCommentEventsForComment(context, issue, comment) {
+    const authorization = await retryAuthorization(context, comment);
+    const event = issueCommentObservation({
+        repository: context.repository,
+        issue,
+        comment,
+        ...(authorization === undefined ? {} : { authorization }),
+        ...(context.adapter === undefined ? {} : { adapter: context.adapter }),
+    });
+    return event === null ? [] : [event];
+}
+async function retryAuthorization(context, comment) {
+    if (comment.body?.trim().toLowerCase() !== '/retry')
+        return undefined;
+    const login = comment.user?.login;
+    if (login === undefined || context.client.collaboratorPermission === undefined)
+        return { source: ReviewerAuthorizationSource.None };
+    try {
+        return {
+            source: ReviewerAuthorizationSource.ProviderPermission,
+            permission: await context.client.collaboratorPermission(context.owner, context.repo, login),
+        };
+    }
+    catch {
+        return { source: ReviewerAuthorizationSource.None };
+    }
+}

package/dist/src/integrations/github/infrastructure/poll-watermark.js

@@ -0,0 +1,22 @@
+export function mergeBatches(batches) {
+    return {
+        drafts: batches.flatMap((batch) => batch.drafts),
+        succeeded: batches.every((batch) => batch.succeeded),
+    };
+}
+export function watermarkCheckpoint(adapter, repository) {
+    return `source:github:${adapter ?? 'github'}:${repository}`;
+}
+export async function loadWatermark(checkpoints, adapter, owner, repo) {
+    if (checkpoints === undefined)
+        return 0;
+    return checkpoints.load(watermarkCheckpoint(adapter, `${owner}/${repo}`));
+}
+export function overlapSince(watermark, overlapMs) {
+    if (!Number.isFinite(watermark) || watermark <= 0)
+        return undefined;
+    return new Date(Math.max(0, watermark - overlapMs)).toISOString();
+}
+export function reportPartialPollFailure(repository, query) {
+    process.stderr.write(`GitHub poll partial failure for ${repository}: ${query}; preserving watermark for replay\n`);
+}

package/dist/src/integrations/github/infrastructure/source-contracts.js

@@ -0,0 +1 @@
+export {};

package/dist/src/integrations/github/infrastructure/source.js

@@ -1,33 +1,37 @@
-import { ReviewerAuthorizationSource, } from '../../../activities/index.js';
 import { GitHubEventType } from '../contracts/events.js';
-import { issueCommentObservation, issueObservation } from './issue-source.js';
+import { issueCommentEventsFor, reviewCommentEventsFor, reviewEventsFor, } from './comment-source.js';
+import { issueObservation } from './issue-source.js';
+import { loadWatermark, overlapSince, reportPartialPollFailure, watermarkCheckpoint, } from './poll-watermark.js';
 import { createGitHubPullRequestSource } from './pr-source.js';
 import { createGitHubRequestCoordinator, } from './request-coordinator.js';
-import { githubReviewObservation } from './review-source.js';
 export function createGitHubSource(config, client, adapter, requests = createGitHubRequestCoordinator({
     maxConcurrent: config.polling.maxConcurrent,
-})) {
-    let nextPollAt = 0;
+}), state) {
     // Draft eventIds are already content fingerprints (see issue-source.ts/pr-source.ts),
     // so the journal itself is idempotent per item. This cache only avoids re-appending
     // (and re-triggering downstream translation) an unchanged item on every poll within
     // a single process lifetime — it's a perf optimization, not a correctness dependency,
     // so it's safe for it to reset on restart.
     const lastEventIds = new Map();
+    const pendingWatermarks = new Map();
     return {
         async poll(signal) {
-            if (Date.now() < nextPollAt)
-                return [];
-            nextPollAt = Date.now() + config.polling.lookbackMs;
-            const perRepository = await Promise.all(config.repositories.map(({ owner, repo }) => pollRepository({
+            const perRepository = await Promise.all(config.repositories.map(async ({ owner, repo }) => pollRepository({
                 client: limitGitHubSourceClient(client, requests),
                 config,
                 adapter,
                 signal,
                 owner,
                 repo,
+                watermark: await loadWatermark(state?.checkpoints, adapter, owner, repo),
+                now: state?.now ?? Date.now,
             })));
-            return perRepository.flat().filter((draft) => {
+            for (const result of perRepository)
+                if (result.succeeded)
+                    pendingWatermarks.set(result.repository, result.completedAt);
+            return perRepository
+                .flatMap((result) => result.drafts)
+                .filter((draft) => {
                 if (draft.eventType !== GitHubEventType.WorkObserved)
                     return true;
                 const prior = lastEventIds.get(draft.payload.externalKey);
@@ -35,17 +39,25 @@ export function createGitHubSource(config, client, adapter, requests = createGit
                 return prior !== draft.eventId;
             });
         },
+        async markPollPersisted() {
+            if (state === undefined)
+                return;
+            for (const [repository, watermark] of pendingWatermarks) {
+                await state.checkpoints.save(watermarkCheckpoint(adapter, repository), watermark);
+                pendingWatermarks.delete(repository);
+            }
+        },
     };
 }
 function limitGitHubSourceClient(client, requests) {
     return {
         ...client,
-        listIssues: (owner, repo, maxResults) => requests.run(() => client.listIssues(owner, repo, maxResults)),
+        listIssues: (owner, repo, maxResults, since) => requests.run(() => client.listIssues(owner, repo, maxResults, since)),
         listPullRequests: (owner, repo, maxResults) => requests.run(() => client.listPullRequests(owner, repo, maxResults)),
         listCheckRunsForRef: (owner, repo, ref) => requests.run(() => client.listCheckRunsForRef(owner, repo, ref)),
         getCombinedStatusForRef: (owner, repo, ref) => requests.run(() => client.getCombinedStatusForRef(owner, repo, ref)),
         listPullRequestFiles: (owner, repo, pullNumber) => requests.run(() => client.listPullRequestFiles(owner, repo, pullNumber)),
-        listIssueComments: (owner, repo, issueNumber, pageSize) => requests.run(() => client.listIssueComments(owner, repo, issueNumber, pageSize)),
+        listIssueComments: (owner, repo, issueNumber, pageSize, since) => requests.run(() => client.listIssueComments(owner, repo, issueNumber, pageSize, since)),
         listReviews: (owner, repo, pullNumber, pageSize) => requests.run(() => client.listReviews(owner, repo, pullNumber, pageSize)),
         ...(client.listReviewComments === undefined
             ? {}
@@ -69,21 +81,39 @@ async function pollRepository(input) {
         repo,
         repository: `${owner}/${repo}`,
     };
-    try {
-        const pullRequestPayloads = await client.listPullRequests(owner, repo, config.polling.maxPerRepo);
-        const [issues, pullRequests, reviews, reviewComments] = await Promise.all([
-            client.listIssues(owner, repo, config.polling.maxPerRepo),
-            createGitHubPullRequestSource({
-                client: { ...client, listPullRequests: async () => pullRequestPayloads },
-                repository: context.repository,
-                maxResults: config.polling.maxPerRepo,
-                ...(adapter === undefined ? {} : { adapter }),
-            }).poll(signal),
-            reviewEventsFor(context, pullRequestPayloads).catch(() => []),
-            reviewCommentEventsFor(context, pullRequestPayloads).catch(() => []),
-        ]);
-        const issueComments = await issueCommentEventsFor(context, issues);
-        return [
+    const since = overlapSince(input.watermark, config.polling.lookbackMs);
+    const [issuesResult, pullRequestsResult] = await Promise.allSettled([
+        client.listIssues(owner, repo, config.polling.maxPerRepo, since),
+        client.listPullRequests(owner, repo, config.polling.maxPerRepo),
+    ]);
+    if (!isFulfilled(issuesResult))
+        reportPartialPollFailure(context.repository, 'issues');
+    if (!isFulfilled(pullRequestsResult))
+        reportPartialPollFailure(context.repository, 'pull requests');
+    const issues = isFulfilled(issuesResult) ? issuesResult.value : [];
+    const pullRequestPayloads = isFulfilled(pullRequestsResult)
+        ? pullRequestsResult.value.filter((pullRequest) => since === undefined || pullRequest.updated_at >= since)
+        : [];
+    const [pullRequests, reviews, reviewComments, issueComments] = await Promise.all([
+        createGitHubPullRequestSource({
+            client: { ...client, listPullRequests: async () => pullRequestPayloads },
+            repository: context.repository,
+            maxResults: config.polling.maxPerRepo,
+            ...(adapter === undefined ? {} : { adapter }),
+        }).poll(signal),
+        reviewEventsFor(context, pullRequestPayloads),
+        reviewCommentEventsFor(context, pullRequestPayloads),
+        issueCommentEventsFor(context, issues, since),
+    ]);
+    return {
+        repository: context.repository,
+        completedAt: input.now(),
+        succeeded: isFulfilled(issuesResult) &&
+            isFulfilled(pullRequestsResult) &&
+            reviews.succeeded &&
+            reviewComments.succeeded &&
+            issueComments.succeeded,
+        drafts: [
             ...issues
                 .filter((issue) => issue.pull_request === undefined)
                 .map((issue) => issueObservation({
@@ -92,67 +122,12 @@ async function pollRepository(input) {
                 ...(adapter === undefined ? {} : { adapter }),
             })),
             ...pullRequests,
-            ...reviews,
-            ...reviewComments,
-            ...issueComments,
-        ];
-    }
-    catch {
-        return [];
-    }
-}
-async function reviewCommentEventsFor(context, pullRequests) {
-    if (context.client.listReviewComments === undefined)
-        return [];
-    const items = await Promise.all(pullRequests.map(async (pullRequest) => (async () => {
-        const comments = await context.client.listReviewComments(context.owner, context.repo, pullRequest.number, context.config.polling.commentPageSize);
-        return (await Promise.all(comments.map((comment) => issueCommentEventsForComment(context, pullRequest, comment)))).flat();
-    })()));
-    return items.flat();
-}
-async function reviewEventsFor(context, pullRequestPayloads) {
-    const items = await Promise.all(pullRequestPayloads.map(async (pullRequest) => {
-        const reviewEvents = await context.client.listReviews(context.owner, context.repo, pullRequest.number, context.config.polling.commentPageSize);
-        return reviewEvents.flatMap((review) => githubReviewObservation({
-            repository: context.repository,
-            pullRequest,
-            review,
-            authorizedReviewers: [],
-        }));
-    }));
-    return items.flat();
-}
-async function issueCommentEventsFor(context, issues) {
-    const items = await Promise.all(issues.map(async (issue) => (async () => {
-        const comments = await context.client.listIssueComments(context.owner, context.repo, issue.number, context.config.polling.commentPageSize);
-        return (await Promise.all(comments.map((comment) => issueCommentEventsForComment(context, issue, comment)))).flat();
-    })()));
-    return items.flat();
-}
-async function issueCommentEventsForComment(context, issue, comment) {
-    const authorization = await retryAuthorization(context, comment);
-    const event = issueCommentObservation({
-        repository: context.repository,
-        issue,
-        comment,
-        ...(authorization === undefined ? {} : { authorization }),
-        ...(context.adapter === undefined ? {} : { adapter: context.adapter }),
-    });
-    return event === null ? [] : [event];
+            ...reviews.drafts,
+            ...reviewComments.drafts,
+            ...issueComments.drafts,
+        ],
+    };
 }
-async function retryAuthorization(context, comment) {
-    if (comment.body?.trim().toLowerCase() !== '/retry')
-        return undefined;
-    const login = comment.user?.login;
-    if (login === undefined || context.client.collaboratorPermission === undefined)
-        return { source: ReviewerAuthorizationSource.None };
-    try {
-        return {
-            source: ReviewerAuthorizationSource.ProviderPermission,
-            permission: await context.client.collaboratorPermission(context.owner, context.repo, login),
-        };
-    }
-    catch {
-        return { source: ReviewerAuthorizationSource.None };
-    }
+function isFulfilled(result) {
+    return 'value' in result;
 }

package/dist/src/integrations/github/provider.js

@@ -27,7 +27,10 @@ export const gitHubProviderDefinition = {
         return {
             adapter,
             eventTypes: Object.values(GitHubEventType),
-            source: createGitHubSource(config, client, adapter, requests),
+            source: createGitHubSource(config, client, adapter, requests, {
+                checkpoints: services.checkpoints,
+                now: () => services.clock.now().getTime(),
+            }),
             maintenance: createGitHubWakeLabelReconciler({
                 orchestration: services.orchestration,
                 resources: services.resources,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atolis-hq/wake",
-  "version": "0.3.45",
+  "version": "0.3.46",
   "description": "Local autonomous agent control plane for software development",
   "license": "Apache-2.0",
   "repository": {