@andypai/agent-kanban 0.5.1 → 0.6.0

package/src/__tests__/postgres-jira-provider.test.ts

@@ -145,6 +145,16 @@ function standardRoutes(opts: { boardCfg?: unknown } = {}): StubRoute[] {
         )
       },
     },
+    {
+      // GET /rest/api/3/issue/{key} backs hydrateIssueByKey's read-after-write.
+      match: (url) => /\/rest\/api\/3\/issue\/ENG-\d+$/.test(new URL(url).pathname),
+      handler: (url) => {
+        const issueKey = new URL(url).pathname.match(/\/issue\/(ENG-\d+)$/)![1]!
+        const issue = issues.find((candidate) => String(candidate.key) === issueKey)
+        if (!issue) return jsonResponse({ errorMessages: ['missing'] }, 404)
+        return jsonResponse(issue)
+      },
+    },
     {
       match: (url) => /\/rest\/api\/3\/issue\/ENG-\d+\/comment$/.test(new URL(url).pathname),
       handler: async (url, init) => {
@@ -401,4 +411,149 @@ describe('postgres jira provider', () => {
       column_id: '20',
     })
   })
+
+  pgTest(
+    'delta sync adds new catalog rows but only a full reconcile prunes obsolete ones',
+    async () => {
+      if (!sql) throw new Error('expected postgres test connection')
+      const prioritiesState = {
+        list: [
+          { id: '2', name: 'High' },
+          { id: '3', name: 'Medium' },
+        ] as Array<{ id: string; name: string }>,
+      }
+      const routes = standardRoutes()
+      const pIdx = routes.findIndex((route) =>
+        route.match('https://example.atlassian.net/rest/api/3/priority'),
+      )
+      routes[pIdx] = {
+        match: (url) => url.includes('/rest/api/3/priority'),
+        handler: () => jsonResponse(prioritiesState.list),
+      }
+      globalThis.fetch = jiraFetchStub(routes).fn
+      // pollingSyncIntervalMs: 0 lets every getBoard run; the first is a full
+      // reconcile (no lastFullSyncAt yet), later ones are delta syncs because the
+      // full-reconcile interval has not elapsed.
+      const provider = new PostgresJiraProvider(sql, {
+        baseUrl: 'https://example.atlassian.net',
+        email: 'user@example.com',
+        apiToken: 'token',
+        projectKey: 'ENG',
+        boardId: 1006,
+        pollingSyncIntervalMs: 0,
+      })
+      const priorityNames = async () =>
+        (await sql!<{ name: string }[]>`SELECT name FROM jira_priorities ORDER BY name`).map(
+          (row) => row.name,
+        )
+
+      await provider.getBoard()
+      expect(await priorityNames()).toEqual(['High', 'Medium'])
+
+      // Drop Medium, add Low upstream. A delta sync must pick up the new row
+      // (Low) via the every-sync UPSERT but must NOT prune the now-obsolete row
+      // (Medium): pruning on a possibly-stale delta snapshot could delete a row
+      // another replica just added. Pruning is confined to the full reconcile.
+      prioritiesState.list = [
+        { id: '2', name: 'High' },
+        { id: '4', name: 'Low' },
+      ]
+      await provider.getBoard()
+      expect(await priorityNames()).toEqual(['High', 'Low', 'Medium'])
+
+      // Force the next sync to be a full reconcile; only then is the obsolete
+      // Medium pruned.
+      await sql`DELETE FROM jira_sync_meta WHERE key = 'lastFullSyncAt'`
+      await provider.getBoard()
+      expect(await priorityNames()).toEqual(['High', 'Low'])
+    },
+  )
+
+  pgTest('concurrent catalog refreshes do not collide on a primary key', async () => {
+    if (!sql) throw new Error('expected postgres test connection')
+    globalThis.fetch = jiraFetchStub(standardRoutes()).fn
+    const sql2 = postgres(databaseUrl as string, { max: 1, onnotice: () => {} })
+    try {
+      const config = {
+        baseUrl: 'https://example.atlassian.net',
+        email: 'user@example.com',
+        apiToken: 'token',
+        projectKey: 'ENG',
+        boardId: 1006,
+        pollingSyncIntervalMs: 0,
+      }
+      // Initialize schema sequentially first; concurrent CREATE TABLE IF NOT
+      // EXISTS races at the DDL level (a pre-existing Postgres quirk unrelated to
+      // catalog refresh). The race under test is the catalog DML below.
+      const p1 = new PostgresJiraProvider(sql, config)
+      await p1.initialize()
+      const p2 = new PostgresJiraProvider(sql2, config)
+      await p2.initialize()
+      // Two replicas on separate connections refreshing the same shared cache
+      // concurrently must not trip a primary-key collision. Both first syncs are
+      // full reconciles (no lastFullSyncAt yet), so this also exercises the
+      // obsolete-row DELETE running concurrently: UPSERT + conditional DELETE is
+      // idempotent and order-independent.
+      await Promise.all([p1.getBoard(), p2.getBoard()])
+      const rows = await sql<
+        { count: number }[]
+      >`SELECT COUNT(*)::int AS count FROM jira_priorities`
+      expect(rows[0]?.count ?? 0).toBeGreaterThan(0)
+    } finally {
+      await sql2.end({ timeout: 1 })
+    }
+  })
+
+  pgTest('full reconcile with a stalled cursor does not prune unfetched-page issues', async () => {
+    if (!sql) throw new Error('expected postgres test connection')
+    // Mirror of the SQLite prune-safety coverage: if the cursor stalls (a
+    // repeated non-last token) mid-scan, seenIssueIds is only partial, so the
+    // incomplete scan must NOT prune issues that exist upstream on pages we never
+    // fetched.
+    let stalled = false
+    const routes = standardRoutes()
+    const sIdx = routes.findIndex((route) => route.match('https://x/rest/api/3/search/jql'))
+    routes[sIdx] = {
+      match: (url) => url.includes('/rest/api/3/search/jql'),
+      handler: () =>
+        stalled
+          ? // Later full reconcile: the cursor stalls after returning only ENG-1,
+            // so ENG-2's page is never reached. isLast=false + a repeated token =>
+            // incomplete.
+            jsonResponse({
+              nextPageToken: 'stuck',
+              isLast: false,
+              issues: [makeIssue({ id: '1', key: 'ENG-1' })],
+            })
+          : // First full reconcile: a clean single terminal page with both issues.
+            jsonResponse({
+              isLast: true,
+              issues: [makeIssue({ id: '1', key: 'ENG-1' }), makeIssue({ id: '2', key: 'ENG-2' })],
+            }),
+    }
+    globalThis.fetch = jiraFetchStub(routes).fn
+    const provider = new PostgresJiraProvider(sql, {
+      baseUrl: 'https://example.atlassian.net',
+      email: 'user@example.com',
+      apiToken: 'token',
+      projectKey: 'ENG',
+      boardId: 1006,
+      pollingSyncIntervalMs: 0,
+    })
+    const issueKeys = async () =>
+      (await sql!<{ key: string }[]>`SELECT key FROM jira_issues ORDER BY key`).map(
+        (row) => row.key,
+      )
+
+    await provider.getBoard()
+    expect(await issueKeys()).toEqual(['ENG-1', 'ENG-2'])
+
+    // Force the next sync to be a full reconcile and hit the stalled cursor.
+    stalled = true
+    await sql`DELETE FROM jira_sync_meta WHERE key = 'lastFullSyncAt'`
+    await provider.getBoard()
+
+    // ENG-2 must survive: the incomplete scan is not authoritative for pruning.
+    expect(await issueKeys()).toEqual(['ENG-1', 'ENG-2'])
+  })
 })

package/src/providers/jira-client.ts

@@ -45,12 +45,73 @@ export interface JiraIssue {
 }
 
 export interface JiraSearchPage {
-  startAt: number
-  maxResults: number
-  total: number
+  // The legacy /rest/api/3/search endpoint returned startAt/maxResults/total.
+  // The current /rest/api/3/search/jql endpoint omits `total` and paginates by
+  // an opaque `nextPageToken` (with `isLast`), so these are optional now.
+  startAt?: number
+  maxResults?: number
+  total?: number
+  nextPageToken?: string
+  isLast?: boolean
   issues: JiraIssue[]
 }
 
+export interface JiraPaginationDecision {
+  // When set, advance the scan to this cursor. When absent, the scan is over and
+  // `complete` says whether it ended definitively (safe to prune against the
+  // accumulated issue set) or was cut short (must NOT prune — issues on unfetched
+  // pages would be wrongly deleted).
+  nextToken?: string
+  complete: boolean
+}
+
+// Decide how a `/rest/api/3/search/jql` cursor scan should proceed after a page.
+// The endpoint signals continuation with `isLast`/`nextPageToken`, but real and
+// degraded servers vary, so termination must be conservative: a scan is reported
+// `complete` (safe to prune against the accumulated issue set) ONLY when the
+// server proves the end, never from a page-size guess.
+//   - usable cursor (fresh `nextPageToken`, isLast !== true) → advance.
+//   - isLast === true                                        → definitive end (complete).
+//   - isLast === false                                       → more pages exist; if the
+//                                                              cursor is missing/stale we
+//                                                              cannot fetch them, so the
+//                                                              scan is incomplete.
+//   - isLast absent, `total` present                         → legacy total/startAt proof:
+//                                                              complete once startAt+count
+//                                                              reaches `total`.
+//   - isLast absent, no `total`, no usable cursor            → NO completeness proof. A
+//                                                              short/empty page must NOT be
+//                                                              read as "the end": a degraded
+//                                                              `{ issues: [] }` would then
+//                                                              prune the entire cache on a
+//                                                              full reconcile. Treat as
+//                                                              incomplete and retry instead.
+// A cursor already in `seenPageTokens` counts as not usable (a stalled cursor
+// that would otherwise loop forever).
+export function decideJiraPagination(
+  page: Pick<JiraSearchPage, 'isLast' | 'nextPageToken' | 'issues' | 'total' | 'startAt'>,
+  seenPageTokens: ReadonlySet<string>,
+): JiraPaginationDecision {
+  const token = page.nextPageToken
+  const canAdvance = !!token && page.isLast !== true && !seenPageTokens.has(token)
+  if (canAdvance) return { nextToken: token, complete: false }
+  if (page.isLast === true) return { complete: true }
+  if (page.isLast === false) return { complete: false }
+  // isLast absent below. Prefer the legacy total/startAt signal when a
+  // (non-standard) server supplies it: complete once we have fetched everything
+  // `total` promises. The live /search/jql endpoint omits `total` and ignores
+  // `startAt`, so the loop never advances by offset — a `total` promising more
+  // than this page is reported incomplete rather than fetched.
+  if (typeof page.total === 'number') {
+    const issueCount = page.issues?.length ?? 0
+    return { complete: (page.startAt ?? 0) + issueCount >= page.total }
+  }
+  // No completeness signal at all (no isLast, no total, no usable cursor). We
+  // cannot prove the scan reached the end, so never guess `complete` from page
+  // size — that would prune the whole cache on a degraded short/empty response.
+  return { complete: false }
+}
+
 export interface JiraCreatePayload {
   fields: Record<string, unknown>
 }
@@ -254,12 +315,17 @@ export class JiraClient {
     startAt: number
     maxResults: number
     fields?: string[]
+    nextPageToken?: string
   }): Promise<JiraSearchPage> {
     const query: QueryParams = {
       jql: params.jql,
-      startAt: params.startAt,
       maxResults: params.maxResults,
     }
+    // /rest/api/3/search/jql ignores startAt and paginates by nextPageToken.
+    // Send the cursor when we have one; only send startAt on the first page for
+    // back-compat with the legacy endpoint.
+    if (params.nextPageToken) query.nextPageToken = params.nextPageToken
+    else query.startAt = params.startAt
     if (params.fields && params.fields.length > 0) {
       query.fields = params.fields.join(',')
     }

package/src/providers/jira.ts

@@ -20,7 +20,13 @@ import {
 import { adfToPlainText, plainTextToAdf, type AdfDocument } from './jira-adf'
 import { JIRA_CAPABILITIES } from './capabilities'
 import { providerUpstreamError, unsupportedOperation } from './errors'
-import { JiraClient, normalizeJiraLabels, type JiraComment, type JiraIssue } from './jira-client'
+import {
+  JiraClient,
+  decideJiraPagination,
+  normalizeJiraLabels,
+  type JiraComment,
+  type JiraIssue,
+} from './jira-client'
 import {
   adjustJiraIssueCommentCount,
   decodeColumnStatusIds,
@@ -114,7 +120,11 @@ export class JiraProvider implements KanbanProvider {
     const lastSyncAtMs = meta.lastSyncAt ? Date.parse(meta.lastSyncAt) : 0
     const now = Date.now()
     if (!force && lastSyncAtMs && now - lastSyncAtMs < this.pollingSyncIntervalMs) return
-    const fullReconcile = force || shouldRunFullReconcile(meta.lastFullSyncAt, now)
+    // `force` only bypasses the poll throttle; it must NOT imply a full
+    // 1970-based reconcile, which re-fetches every issue plus a per-issue
+    // changelog call (~minutes). Writes get read-after-write freshness from
+    // hydrateIssueByKey instead, so a forced sync stays a cheap delta.
+    const fullReconcile = shouldRunFullReconcile(meta.lastFullSyncAt, now)
 
     // 1. Resolve project.
     const project = await this.client.getProject(this.config.projectKey)
@@ -181,10 +191,7 @@ export class JiraProvider implements KanbanProvider {
     const sinceClause = since ?? '1970-01-01 00:00'
     const jql = `project = ${project.key} AND updated >= "${sinceClause}" ORDER BY updated ASC`
 
-    let startAt = 0
     const maxResults = 100
-    let accumulated = 0
-    let total = Infinity
     let newestUpdatedAt: string | null = meta.lastIssueUpdatedAt
     const seenIssueIds = new Set<string>()
     const issueFields = [
@@ -200,12 +207,28 @@ export class JiraProvider implements KanbanProvider {
       'updated',
       'project',
     ]
-    // Terminates when accumulated reaches total, or when the server returns
-    // an empty page (defensive against buggy servers not advancing startAt).
-    while (accumulated < total) {
-      const page = await this.client.listIssues({ jql, startAt, maxResults, fields: issueFields })
-      total = page.total
-      if (page.issues.length === 0) break
+    // /rest/api/3/search/jql paginates by an opaque nextPageToken and omits
+    // `total`, so we follow the cursor until the server reports `isLast` or stops
+    // handing back a token. The previous total/startAt loop fetched only the first
+    // page (oldest 100 by updated ASC) once `total` came back undefined, so every
+    // issue beyond the first page — including newly-created tickets on a project
+    // with >100 issues — was never cached. seenPageTokens guards against a server
+    // that repeats a cursor, which would otherwise spin this loop forever and hang
+    // the poll cycle; an empty page is not treated as terminal because a non-last
+    // page may legitimately carry a token with zero issues.
+    const seenPageTokens = new Set<string>()
+    let nextPageToken: string | undefined
+    let firstPage = true
+    let paginationComplete = false
+    while (firstPage || nextPageToken !== undefined) {
+      firstPage = false
+      const page = await this.client.listIssues({
+        jql,
+        startAt: 0,
+        maxResults,
+        fields: issueFields,
+        nextPageToken,
+      })
 
       upsertJiraIssues(
         this.db,
@@ -249,10 +272,37 @@ export class JiraProvider implements KanbanProvider {
         })
       }
 
-      accumulated += page.issues.length
-      startAt += page.issues.length
+      const decision = decideJiraPagination(page, seenPageTokens)
+      if (decision.nextToken !== undefined) {
+        seenPageTokens.add(decision.nextToken)
+        nextPageToken = decision.nextToken
+        continue
+      }
+      paginationComplete = decision.complete
+      if (!paginationComplete) {
+        // The server stopped advancing the cursor before reporting a definitive
+        // end (stalled/contradictory). seenIssueIds is only a partial scan, so we
+        // must not treat it as authoritative for pruning below.
+        console.warn(
+          `[jira] search/jql scan ended without a definitive last page; treating as incomplete`,
+        )
+      }
+      break
     }
 
+    if (!paginationComplete) {
+      // The scan ended early (stalled/contradictory cursor). Leave sync metadata
+      // unchanged so this partial result is not recorded as a clean sync: lastSyncAt
+      // is not advanced, so the next sync is not throttled and retries promptly, and
+      // the full-reconcile marker stays due. The issues we did fetch are already
+      // cached (additive); we only skip pruning — which would delete issues that
+      // exist upstream on pages we never fetched — and the metadata advance.
+      return
+    }
+
+    // Prune against seenIssueIds and advance lastFullSyncAt only on a full
+    // reconcile; a delta sync's seenIssueIds is intentionally partial. (The scan
+    // is known complete here — an incomplete scan returned above.)
     if (fullReconcile) {
       pruneJiraIssuesMissingUpstream(this.db, project.key, [...seenIssueIds])
     }
@@ -452,6 +502,55 @@ export class JiraProvider implements KanbanProvider {
     }
   }
 
+  // Read-after-write via the direct issue endpoint. Unlike JQL search (used by
+  // sync()), GET /issue/{key} has no search-index lag, so a just-created or
+  // just-transitioned issue is reflected immediately. This replaces the previous
+  // "create/transition then sync(true) then getCachedTask" pattern, which raced
+  // the search index (creates reported "not yet visible"; a move's new status
+  // sometimes didn't land) and forced a full whole-project reconcile per write.
+  private async hydrateIssueByKey(key: string): Promise<Task> {
+    // getIssue throws on a missing key (404), so reaching this method means the
+    // issue exists upstream. A null read-back would therefore be a genuine cache
+    // anomaly (the upsert above did not land), not an ordinary not-found — surface
+    // it rather than threading an unreachable null through every caller.
+    const issue = await this.client.getIssue(key)
+    upsertJiraIssues(this.db, [
+      {
+        id: issue.id,
+        key: issue.key,
+        summary: issue.fields.summary,
+        descriptionText: issue.fields.description
+          ? adfToPlainText(issue.fields.description as AdfDocument)
+          : '',
+        statusId: issue.fields.status.id,
+        priorityName: issue.fields.priority?.name ?? null,
+        issueTypeName: issue.fields.issuetype?.name ?? '',
+        assigneeAccountId: issue.fields.assignee?.accountId ?? null,
+        assigneeName: issue.fields.assignee?.displayName ?? null,
+        labels: issue.fields.labels ?? [],
+        commentCount: issue.fields.comment?.total ?? 0,
+        projectKey: issue.fields.project?.key ?? this.config.projectKey,
+        url: `${this.config.baseUrl}/browse/${issue.key}`,
+        createdAt: issue.fields.created,
+        updatedAt: issue.fields.updated,
+      },
+    ])
+    // Ingest the changelog like the sync loop does, so a just-applied transition
+    // is recorded in jira_activity immediately (backs getActivity and the
+    // poll-based `moved` trigger) rather than waiting for the next unthrottled
+    // sync. Best-effort: activity must not fail the mutation.
+    await this.ingestIssueActivity(issue.id).catch((err) => {
+      console.warn(`[jira] activity fetch for ${issue.key} failed:`, err)
+    })
+    const task = getCachedTask(this.db, key)
+    if (!task) {
+      providerUpstreamError(
+        `Jira issue ${key} was hydrated from GET /issue but is missing from the cache.`,
+      )
+    }
+    return task
+  }
+
   async createTask(input: CreateTaskInput): Promise<Task> {
     await this.sync()
     this.normalizeProjectField(input.project)
@@ -479,14 +578,7 @@ export class JiraProvider implements KanbanProvider {
     // issues land in the project workflow's default start state. Use
     // `moveTask` after create to change status.
     const created = await this.client.createIssue({ fields })
-    await this.sync(true)
-    const fresh = getCachedTask(this.db, created.key)
-    if (!fresh) {
-      providerUpstreamError(
-        `Jira issue ${created.key} was created but is not yet visible in the cache after sync.`,
-      )
-    }
-    return fresh
+    return this.hydrateIssueByKey(created.key)
   }
 
   async updateTask(idOrRef: string, input: UpdateTaskInput): Promise<Task> {
@@ -521,12 +613,7 @@ export class JiraProvider implements KanbanProvider {
     if (Object.keys(fields).length > 0) {
       await this.client.updateIssue(issueKey, { fields })
     }
-    await this.sync(true)
-    const fresh = getCachedTask(this.db, issueKey)
-    if (!fresh) {
-      providerUpstreamError(`Jira issue ${issueKey} disappeared from cache after update.`)
-    }
-    return fresh
+    return this.hydrateIssueByKey(issueKey)
   }
 
   async moveTask(idOrRef: string, column: string): Promise<Task> {
@@ -563,12 +650,7 @@ export class JiraProvider implements KanbanProvider {
       )
     }
     await this.client.transitionIssue(issueKey, match.id)
-    await this.sync(true)
-    const fresh = getCachedTask(this.db, issueKey)
-    if (!fresh) {
-      providerUpstreamError(`Jira issue ${issueKey} missing from cache after transition.`)
-    }
-    return fresh
+    return this.hydrateIssueByKey(issueKey)
   }
 
   async deleteTask(_idOrRef: string): Promise<Task> {