@sunilp-org/jam-cli 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +304 -52
  2. package/dist/commands/ask.d.ts +4 -0
  3. package/dist/commands/ask.d.ts.map +1 -1
  4. package/dist/commands/ask.js +202 -10
  5. package/dist/commands/ask.js.map +1 -1
  6. package/dist/commands/commit.d.ts +12 -0
  7. package/dist/commands/commit.d.ts.map +1 -0
  8. package/dist/commands/commit.js +135 -0
  9. package/dist/commands/commit.js.map +1 -0
  10. package/dist/commands/context.d.ts +12 -0
  11. package/dist/commands/context.d.ts.map +1 -0
  12. package/dist/commands/context.js +52 -0
  13. package/dist/commands/context.js.map +1 -0
  14. package/dist/commands/review.d.ts +25 -0
  15. package/dist/commands/review.d.ts.map +1 -0
  16. package/dist/commands/review.js +117 -0
  17. package/dist/commands/review.js.map +1 -0
  18. package/dist/commands/run.d.ts +1 -0
  19. package/dist/commands/run.d.ts.map +1 -1
  20. package/dist/commands/run.js +199 -197
  21. package/dist/commands/run.js.map +1 -1
  22. package/dist/config/loader.d.ts.map +1 -1
  23. package/dist/config/loader.js +3 -0
  24. package/dist/config/loader.js.map +1 -1
  25. package/dist/index.js +63 -1
  26. package/dist/index.js.map +1 -1
  27. package/dist/providers/base.d.ts +26 -0
  28. package/dist/providers/base.d.ts.map +1 -1
  29. package/dist/providers/factory.d.ts.map +1 -1
  30. package/dist/providers/factory.js +17 -1
  31. package/dist/providers/factory.js.map +1 -1
  32. package/dist/providers/groq.d.ts +16 -0
  33. package/dist/providers/groq.d.ts.map +1 -0
  34. package/dist/providers/groq.js +23 -0
  35. package/dist/providers/groq.js.map +1 -0
  36. package/dist/providers/ollama.d.ts +6 -1
  37. package/dist/providers/ollama.d.ts.map +1 -1
  38. package/dist/providers/ollama.js +77 -4
  39. package/dist/providers/ollama.js.map +1 -1
  40. package/dist/providers/openai.d.ts +18 -0
  41. package/dist/providers/openai.d.ts.map +1 -0
  42. package/dist/providers/openai.js +229 -0
  43. package/dist/providers/openai.js.map +1 -0
  44. package/dist/tools/all-tools.d.ts +18 -0
  45. package/dist/tools/all-tools.d.ts.map +1 -0
  46. package/dist/tools/all-tools.js +95 -0
  47. package/dist/tools/all-tools.js.map +1 -0
  48. package/dist/tools/apply_patch.js +1 -1
  49. package/dist/tools/apply_patch.js.map +1 -1
  50. package/dist/tools/context-tools.d.ts +14 -0
  51. package/dist/tools/context-tools.d.ts.map +1 -0
  52. package/dist/tools/context-tools.js +63 -0
  53. package/dist/tools/context-tools.js.map +1 -0
  54. package/dist/tools/git_diff.js +1 -1
  55. package/dist/tools/git_diff.js.map +1 -1
  56. package/dist/tools/git_status.js +1 -1
  57. package/dist/tools/git_status.js.map +1 -1
  58. package/dist/tools/registry.d.ts.map +1 -1
  59. package/dist/tools/registry.js +2 -0
  60. package/dist/tools/registry.js.map +1 -1
  61. package/dist/tools/run_command.d.ts +8 -3
  62. package/dist/tools/run_command.d.ts.map +1 -1
  63. package/dist/tools/run_command.js +90 -3
  64. package/dist/tools/run_command.js.map +1 -1
  65. package/dist/ui/chat.d.ts.map +1 -1
  66. package/dist/ui/chat.js +173 -1
  67. package/dist/ui/chat.js.map +1 -1
  68. package/dist/ui/logo.d.ts.map +1 -1
  69. package/dist/ui/logo.js +5 -1
  70. package/dist/ui/logo.js.map +1 -1
  71. package/dist/utils/agent.d.ts +130 -0
  72. package/dist/utils/agent.d.ts.map +1 -0
  73. package/dist/utils/agent.js +449 -0
  74. package/dist/utils/agent.js.map +1 -0
  75. package/dist/utils/cache.d.ts +30 -0
  76. package/dist/utils/cache.d.ts.map +1 -0
  77. package/dist/utils/cache.js +62 -0
  78. package/dist/utils/cache.js.map +1 -0
  79. package/dist/utils/context.d.ts +38 -0
  80. package/dist/utils/context.d.ts.map +1 -0
  81. package/dist/utils/context.js +383 -0
  82. package/dist/utils/context.js.map +1 -0
  83. package/dist/utils/critic.d.ts +31 -0
  84. package/dist/utils/critic.d.ts.map +1 -0
  85. package/dist/utils/critic.js +126 -0
  86. package/dist/utils/critic.js.map +1 -0
  87. package/dist/utils/index-builder.d.ts +53 -0
  88. package/dist/utils/index-builder.d.ts.map +1 -0
  89. package/dist/utils/index-builder.js +241 -0
  90. package/dist/utils/index-builder.js.map +1 -0
  91. package/dist/utils/memory.d.ts +104 -0
  92. package/dist/utils/memory.d.ts.map +1 -0
  93. package/dist/utils/memory.js +215 -0
  94. package/dist/utils/memory.js.map +1 -0
  95. package/dist/utils/past-sessions.d.ts +31 -0
  96. package/dist/utils/past-sessions.d.ts.map +1 -0
  97. package/dist/utils/past-sessions.js +126 -0
  98. package/dist/utils/past-sessions.js.map +1 -0
  99. package/dist/utils/tokens.d.ts +53 -0
  100. package/dist/utils/tokens.d.ts.map +1 -0
  101. package/dist/utils/tokens.js +138 -0
  102. package/dist/utils/tokens.js.map +1 -0
  103. package/package.json +4 -2
@@ -0,0 +1,126 @@
1
+ /**
2
+ * Past session search — find relevant Q&A from previous sessions.
3
+ *
4
+ * Uses a simple TF-IDF-like keyword overlap to find past conversations
5
+ * that are relevant to the current question. No vector embeddings needed —
6
+ * this is a pragmatic approach for local-first CLI tools.
7
+ */
8
+ import { listSessions, getSession } from '../storage/history.js';
9
+ // ── Tokenization / scoring ────────────────────────────────────────────────────
10
+ const STOP_WORDS = new Set([
11
+ 'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
12
+ 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
13
+ 'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
14
+ 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'about',
15
+ 'this', 'that', 'and', 'or', 'but', 'not', 'so', 'if', 'then',
16
+ 'than', 'too', 'very', 'just', 'how', 'where', 'what', 'which',
17
+ 'who', 'when', 'why', 'all', 'each', 'every', 'some', 'any',
18
+ 'its', 'it', 'you', 'your', 'we', 'our', 'they', 'their',
19
+ 'i', 'me', 'my', 'change', 'adding', 'using', 'make', 'use',
20
+ ]);
21
+ function tokenize(text) {
22
+ return text
23
+ .toLowerCase()
24
+ .replace(/[^a-z0-9\s_.-]/g, ' ')
25
+ .split(/\s+/)
26
+ .filter(w => w.length > 2 && !STOP_WORDS.has(w));
27
+ }
28
+ /**
29
+ * Compute a simple keyword overlap score between two texts.
30
+ * Returns 0–1 where 1 means perfect overlap.
31
+ */
32
+ function keywordOverlap(queryTokens, targetTokens) {
33
+ if (queryTokens.length === 0 || targetTokens.length === 0)
34
+ return 0;
35
+ const targetSet = new Set(targetTokens);
36
+ const matches = queryTokens.filter(t => targetSet.has(t));
37
+ // Jaccard-like: overlap / union
38
+ const union = new Set([...queryTokens, ...targetTokens]);
39
+ return matches.length / union.size;
40
+ }
41
+ // ── Main search ───────────────────────────────────────────────────────────────
42
+ /**
43
+ * Search past sessions for Q&A exchanges relevant to the current question.
44
+ *
45
+ * @param question The user's current question.
46
+ * @param workspaceRoot Current workspace root (to scope sessions).
47
+ * @param maxResults Maximum exchanges to return.
48
+ * @param minScore Minimum relevance score to include.
49
+ */
50
+ export async function searchPastSessions(question, workspaceRoot, maxResults = 3, minScore = 0.15) {
51
+ const queryTokens = tokenize(question);
52
+ if (queryTokens.length === 0)
53
+ return [];
54
+ let sessions;
55
+ try {
56
+ sessions = await listSessions();
57
+ }
58
+ catch {
59
+ return [];
60
+ }
61
+ // Only look at sessions from the same workspace, limit to recent 20
62
+ const relevantSessions = sessions
63
+ .filter(s => s.workspaceRoot === workspaceRoot)
64
+ .slice(0, 20);
65
+ const candidates = [];
66
+ for (const sessionMeta of relevantSessions) {
67
+ let session;
68
+ try {
69
+ session = await getSession(sessionMeta.id);
70
+ }
71
+ catch {
72
+ continue;
73
+ }
74
+ if (!session?.messages)
75
+ continue;
76
+ // Extract user→assistant pairs
77
+ for (let i = 0; i < session.messages.length - 1; i++) {
78
+ const msg = session.messages[i];
79
+ const next = session.messages[i + 1];
80
+ if (msg.role === 'user' && next?.role === 'assistant') {
81
+ // Skip tool-result injections and system messages
82
+ if (msg.content.startsWith('[Tool result:') || msg.content.startsWith('[SYSTEM'))
83
+ continue;
84
+ if (msg.content.startsWith('[WORKING MEMORY') || msg.content.startsWith('[CONTEXT'))
85
+ continue;
86
+ const questionTokens = tokenize(msg.content);
87
+ const score = keywordOverlap(queryTokens, questionTokens);
88
+ if (score >= minScore) {
89
+ candidates.push({
90
+ question: msg.content.slice(0, 500),
91
+ answer: next.content.slice(0, 1500), // Cap size
92
+ sessionId: session.id,
93
+ score,
94
+ });
95
+ }
96
+ }
97
+ }
98
+ }
99
+ // Sort by score descending, take top N
100
+ return candidates
101
+ .sort((a, b) => b.score - a.score)
102
+ .slice(0, maxResults);
103
+ }
104
+ /**
105
+ * Format past exchanges as context to inject into the system prompt or messages.
106
+ */
107
+ export function formatPastExchanges(exchanges) {
108
+ if (exchanges.length === 0)
109
+ return '';
110
+ const parts = [
111
+ '## Relevant Past Conversations',
112
+ '',
113
+ 'These previous Q&A exchanges from this project may provide useful context:',
114
+ '',
115
+ ];
116
+ for (const ex of exchanges) {
117
+ parts.push(`**Q:** ${ex.question.slice(0, 200)}`);
118
+ parts.push(`**A:** ${ex.answer.slice(0, 500)}`);
119
+ parts.push('');
120
+ }
121
+ parts.push('---');
122
+ parts.push('Use the above as background context, but always verify by reading current code.');
123
+ parts.push('');
124
+ return parts.join('\n');
125
+ }
126
+ //# sourceMappingURL=past-sessions.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"past-sessions.js","sourceRoot":"","sources":["../../src/utils/past-sessions.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAgBjE,iFAAiF;AAEjF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IACnE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IACnE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;IACjE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO;IAClE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM;IAC7D,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO;IAC9D,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;IAC3D,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IACxD,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;CAC5D,CAAC,CAAC;AAEH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,WAAqB,EAAE,YAAsB;IACnE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEpE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1D,gCAAgC;IAChC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,WAAW,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC;IACzD,OAAO,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;AACrC,CAAC;AAED,iFAAiF;AAEjF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,aAAqB,EACrB,aAAqB,CAAC,EACtB,WAAmB,IAAI;IAEvB,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAExC,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,oEAAoE;IACpE,MAAM,gBAAgB,GAAG,QAAQ;SAC9B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,aAAa,CAAC;SAC9C,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,MAAM,UAAU,GAAmB,EAAE,CAAC;IAEtC,KAAK,MAAM,WAAW,IAAI,gBAAgB,EAAE,CAAC;QAC3C,IAAI,OAA6B,CAAC;QAClC,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,QAAQ;YAAE,SAAS;QAEjC,+BAA+B;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACrD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC;YACjC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAErC,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,EAAE,IAAI,KAAK,WAAW,EAAE,CAAC;gBACtD,kDAAkD;gBAClD,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;oBAAE,SAAS;gBAC3F,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC;oBAAE,SAAS;gBAE9F,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC7C,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBAE1D,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;oBACtB,UAAU,CAAC,IAAI,CAAC;wBACd,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;wBACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,WAAW;wBAChD,SAAS,EAAE,OAAO,CAAC,EAAE;wBACrB,KAAK;qBACN,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,uCAAuC;IACvC,OAAO,UAAU;SACd,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAAyB;IAC3D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEtC,MAAM,KAAK,GAAG;QACZ,gCAAgC;QAChC,EAAE;QACF,4EAA4E;QAC5E,EAAE;KACH,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;IAC9F,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
@@ -0,0 +1,53 @@
1
+ /**
2
+ * Token estimation utilities for context window management.
3
+ *
4
+ * Uses a heuristic character-to-token ratio rather than a full tokenizer
5
+ * to keep dependencies minimal. The ratios are calibrated for typical
6
+ * code / English text seen by llama-family models (~3.5–4 chars per token).
7
+ *
8
+ * The key concern is *budget management* — we don't need exact counts,
9
+ * just good-enough estimates to decide when to evict or summarize.
10
+ */
11
+ /**
12
+ * Known context-window sizes (tokens) for popular local models.
13
+ * Used as fallback when the provider doesn't report a limit.
14
+ */
15
+ export declare const MODEL_CONTEXT_LIMITS: Record<string, number>;
16
+ /** Rough token count for a string. */
17
+ export declare function estimateTokens(text: string): number;
18
+ /** Estimate tokens for an array of messages (role + content). */
19
+ export declare function estimateMessageTokens(messages: Array<{
20
+ role: string;
21
+ content: string;
22
+ }>): number;
23
+ /**
24
+ * Get the effective context budget for a model (in tokens).
25
+ * This is the max tokens we allow in the message array before
26
+ * triggering summarization / eviction.
27
+ */
28
+ export declare function getContextBudget(model?: string): number;
29
+ /**
30
+ * Check whether the current messages exceed the context budget.
31
+ * Returns { overBudget, currentTokens, budget, excess }.
32
+ */
33
+ export declare function checkBudget(messages: Array<{
34
+ role: string;
35
+ content: string;
36
+ }>, systemPrompt: string | undefined, model?: string): {
37
+ overBudget: boolean;
38
+ currentTokens: number;
39
+ budget: number;
40
+ excess: number;
41
+ };
42
+ /**
43
+ * Truncate text to fit within a token budget, keeping the beginning and end
44
+ * (most useful context is usually at boundaries).
45
+ */
46
+ export declare function truncateToTokenBudget(text: string, maxTokens: number): string;
47
+ /**
48
+ * Truncate tool output (file contents, search results) to a sensible size.
49
+ * - File reads: keep first + last N lines
50
+ * - Search results: keep first N matches
51
+ */
52
+ export declare function truncateToolOutput(toolName: string, output: string, maxTokens?: number): string;
53
+ //# sourceMappingURL=tokens.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../../src/utils/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAOH;;;GAGG;AACH,eAAO,MAAM,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAgBvD,CAAC;AAaF,sCAAsC;AACtC,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED,iEAAiE;AACjE,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,MAAM,CAOhG;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAUvD;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,QAAQ,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,EAClD,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,KAAK,CAAC,EAAE,MAAM,GACb;IAAE,UAAU,EAAE,OAAO,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAahF;AAID;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAY7E;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,SAAS,GAAE,MAAa,GACvB,MAAM,CA2BR"}
@@ -0,0 +1,138 @@
1
+ /**
2
+ * Token estimation utilities for context window management.
3
+ *
4
+ * Uses a heuristic character-to-token ratio rather than a full tokenizer
5
+ * to keep dependencies minimal. The ratios are calibrated for typical
6
+ * code / English text seen by llama-family models (~3.5–4 chars per token).
7
+ *
8
+ * The key concern is *budget management* — we don't need exact counts,
9
+ * just good-enough estimates to decide when to evict or summarize.
10
+ */
11
+ // ── Constants ─────────────────────────────────────────────────────────────────
12
+ /** Average characters per token for English + code. */
13
+ const CHARS_PER_TOKEN = 3.8;
14
+ /**
15
+ * Known context-window sizes (tokens) for popular local models.
16
+ * Used as fallback when the provider doesn't report a limit.
17
+ */
18
+ export const MODEL_CONTEXT_LIMITS = {
19
+ 'llama3.2': 128_000,
20
+ 'llama3.2:1b': 8_192,
21
+ 'llama3.2:3b': 128_000,
22
+ 'llama3.1': 128_000,
23
+ 'llama3': 8_192,
24
+ 'llama2': 4_096,
25
+ 'mistral': 8_192,
26
+ 'mixtral': 32_768,
27
+ 'codellama': 16_384,
28
+ 'deepseek-coder': 16_384,
29
+ 'deepseek-coder-v2': 128_000,
30
+ 'qwen2.5-coder': 128_000,
31
+ 'phi3': 128_000,
32
+ 'gemma2': 8_192,
33
+ 'command-r': 128_000,
34
+ };
35
+ /** Safe default when model is unknown (conservative). */
36
+ const DEFAULT_CONTEXT_LIMIT = 8_192;
37
+ /**
38
+ * How much of the context window we're willing to fill with messages.
39
+ * Leave headroom for the model's own generation and system prompt.
40
+ */
41
+ const USAGE_RATIO = 0.75;
42
+ // ── Estimation functions ──────────────────────────────────────────────────────
43
+ /** Rough token count for a string. */
44
+ export function estimateTokens(text) {
45
+ if (!text)
46
+ return 0;
47
+ return Math.ceil(text.length / CHARS_PER_TOKEN);
48
+ }
49
+ /** Estimate tokens for an array of messages (role + content). */
50
+ export function estimateMessageTokens(messages) {
51
+ let total = 0;
52
+ for (const msg of messages) {
53
+ // ~4 tokens overhead per message for role, delimiters
54
+ total += 4 + estimateTokens(msg.content);
55
+ }
56
+ return total;
57
+ }
58
+ /**
59
+ * Get the effective context budget for a model (in tokens).
60
+ * This is the max tokens we allow in the message array before
61
+ * triggering summarization / eviction.
62
+ */
63
+ export function getContextBudget(model) {
64
+ if (!model)
65
+ return Math.floor(DEFAULT_CONTEXT_LIMIT * USAGE_RATIO);
66
+ // Try exact match first, then prefix match
67
+ const lower = model.toLowerCase();
68
+ const limit = MODEL_CONTEXT_LIMITS[lower]
69
+ ?? Object.entries(MODEL_CONTEXT_LIMITS).find(([k]) => lower.startsWith(k))?.[1]
70
+ ?? DEFAULT_CONTEXT_LIMIT;
71
+ return Math.floor(limit * USAGE_RATIO);
72
+ }
73
+ /**
74
+ * Check whether the current messages exceed the context budget.
75
+ * Returns { overBudget, currentTokens, budget, excess }.
76
+ */
77
+ export function checkBudget(messages, systemPrompt, model) {
78
+ const budget = getContextBudget(model);
79
+ const systemTokens = systemPrompt ? estimateTokens(systemPrompt) + 4 : 0;
80
+ const msgTokens = estimateMessageTokens(messages);
81
+ const currentTokens = systemTokens + msgTokens;
82
+ const excess = currentTokens - budget;
83
+ return {
84
+ overBudget: excess > 0,
85
+ currentTokens,
86
+ budget,
87
+ excess: Math.max(0, excess),
88
+ };
89
+ }
90
+ // ── Text truncation helpers ───────────────────────────────────────────────────
91
+ /**
92
+ * Truncate text to fit within a token budget, keeping the beginning and end
93
+ * (most useful context is usually at boundaries).
94
+ */
95
+ export function truncateToTokenBudget(text, maxTokens) {
96
+ const estimated = estimateTokens(text);
97
+ if (estimated <= maxTokens)
98
+ return text;
99
+ const maxChars = Math.floor(maxTokens * CHARS_PER_TOKEN);
100
+ const keepChars = Math.floor(maxChars * 0.45); // 45% from start, 45% from end, ~10% for marker
101
+ const head = text.slice(0, keepChars);
102
+ const tail = text.slice(-keepChars);
103
+ const omittedTokens = estimated - maxTokens;
104
+ return `${head}\n\n[… ~${omittedTokens} tokens omitted …]\n\n${tail}`;
105
+ }
106
+ /**
107
+ * Truncate tool output (file contents, search results) to a sensible size.
108
+ * - File reads: keep first + last N lines
109
+ * - Search results: keep first N matches
110
+ */
111
+ export function truncateToolOutput(toolName, output, maxTokens = 1500) {
112
+ const estimated = estimateTokens(output);
113
+ if (estimated <= maxTokens)
114
+ return output;
115
+ if (toolName === 'read_file') {
116
+ // For file reads, keep head + tail for best context
117
+ const lines = output.split('\n');
118
+ const maxLines = Math.floor(maxTokens / 10); // ~10 tokens per line avg
119
+ if (lines.length <= maxLines)
120
+ return output;
121
+ const keepLines = Math.floor(maxLines * 0.45);
122
+ const head = lines.slice(0, keepLines).join('\n');
123
+ const tail = lines.slice(-keepLines).join('\n');
124
+ const omitted = lines.length - keepLines * 2;
125
+ return `${head}\n\n[… ${omitted} lines omitted …]\n\n${tail}`;
126
+ }
127
+ if (toolName === 'search_text') {
128
+ // For search results, keep first N results (most relevant)
129
+ const lines = output.split('\n');
130
+ const maxLines = Math.floor(maxTokens / 8);
131
+ if (lines.length <= maxLines)
132
+ return output;
133
+ return lines.slice(0, maxLines).join('\n') + `\n\n[… ${lines.length - maxLines} more results truncated]`;
134
+ }
135
+ // Generic truncation
136
+ return truncateToTokenBudget(output, maxTokens);
137
+ }
138
+ //# sourceMappingURL=tokens.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/utils/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,iFAAiF;AAEjF,uDAAuD;AACvD,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAA2B;IAC1D,UAAU,EAAS,OAAO;IAC1B,aAAa,EAAQ,KAAK;IAC1B,aAAa,EAAM,OAAO;IAC1B,UAAU,EAAS,OAAO;IAC1B,QAAQ,EAAa,KAAK;IAC1B,QAAQ,EAAa,KAAK;IAC1B,SAAS,EAAY,KAAK;IAC1B,SAAS,EAAW,MAAM;IAC1B,WAAW,EAAS,MAAM;IAC1B,gBAAgB,EAAI,MAAM;IAC1B,mBAAmB,EAAC,OAAO;IAC3B,eAAe,EAAI,OAAO;IAC1B,MAAM,EAAc,OAAO;IAC3B,QAAQ,EAAc,KAAK;IAC3B,WAAW,EAAS,OAAO;CAC5B,CAAC;AAEF,yDAAyD;AACzD,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAEpC;;;GAGG;AACH,MAAM,WAAW,GAAG,IAAI,CAAC;AAEzB,iFAAiF;AAEjF,sCAAsC;AACtC,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC;AAClD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,qBAAqB,CAAC,QAAkD;IACtF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,sDAAsD;QACtD,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,qBAAqB,GAAG,WAAW,CAAC,CAAC;IAEnE,2CAA2C;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,oBAAoB,CAAC,KAAK,CAAC;WACpC,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;WAC5E,qBAAqB,CAAC;IAE3B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,WAAW,CAAC,CAAC;AACzC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,QAAkD,EAClD,YAAgC,EAChC,KAAc;IAEd,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,MAAM,SAAS,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,aAAa,GAAG,YAAY,GAAG,SAAS,CAAC;IAC/C,MAAM,MAAM,GAAG,aAAa,GAAG,MAAM,CAAC;IAEtC,OAAO;QACL,UAAU,EAAE,MAAM,GAAG,CAAC;QACtB,aAAa;QACb,MAAM;QACN,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,iFAAiF;AAEjF;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,SAAiB;IACnE,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,SAAS,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,eAAe,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,gDAAgD;IAE/F,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,aAAa,GAAG,SAAS,GAAG,SAAS,CAAC;IAE5C,OAAO,GAAG,IAAI,WAAW,aAAa,yBAAyB,IAAI,EAAE,CAAC;AACxE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,QAAgB,EAChB,MAAc,EACd,YAAoB,IAAI;IAExB,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,SAAS,IAAI,SAAS;QAAE,OAAO,MAAM,CAAC;IAE1C,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;QAC7B,oDAAoD;QACpD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,0BAA0B;QACvE,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;YAAE,OAAO,MAAM,CAAC;QAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC;QAC7C,OAAO,GAAG,IAAI,UAAU,OAAO,wBAAwB,IAAI,EAAE,CAAC;IAChE,CAAC;IAED,IAAI,QAAQ,KAAK,aAAa,EAAE,CAAC;QAC/B,2DAA2D;QAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QAC3C,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;YAAE,OAAO,MAAM,CAAC;QAC5C,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,UAAU,KAAK,CAAC,MAAM,GAAG,QAAQ,0BAA0B,CAAC;IAC3G,CAAC;IAED,qBAAqB;IACrB,OAAO,qBAAqB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAClD,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sunilp-org/jam-cli",
3
- "version": "0.1.0",
3
+ "version": "0.1.1",
4
4
  "description": "Jam — developer-first AI assistant CLI for the terminal. Ask questions, explain code, review diffs, generate patches, and run agentic tasks powered by Ollama.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -53,13 +53,15 @@
53
53
  "test:coverage": "vitest run --coverage",
54
54
  "clean": "rm -rf dist"
55
55
  },
56
+ "optionalDependencies": {
57
+ "keytar": "^7.9.0"
58
+ },
56
59
  "dependencies": {
57
60
  "chalk": "^5.3.0",
58
61
  "commander": "^12.1.0",
59
62
  "cosmiconfig": "^9.0.0",
60
63
  "ink": "^5.0.1",
61
64
  "ink-text-input": "^6.0.0",
62
- "keytar": "^7.9.0",
63
65
  "marked": "^12.0.0",
64
66
  "marked-terminal": "^7.1.0",
65
67
  "minimatch": "^9.0.5",