@pi-in-go/pigpen-session-ingest 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CREDITS.md +15 -0
- package/LICENSE +21 -0
- package/README.md +50 -0
- package/extensions/session-ingest/bench_test.go +68 -0
- package/extensions/session-ingest/extension.go +1111 -0
- package/extensions/session-ingest/extension_test.go +65 -0
- package/extensions/session-ingest/fakehost_test.go +548 -0
- package/extensions/session-ingest/go.mod +6 -0
- package/extensions/session-ingest/go.sum +2 -0
- package/extensions/session-ingest/modes.go +464 -0
- package/extensions/session-ingest/modes_test.go +254 -0
- package/extensions/session-ingest/parse_allocs_test.go +36 -0
- package/extensions/session-ingest/review_test.go +53 -0
- package/extensions/session-ingest/session_test.go +267 -0
- package/package.json +38 -0
- package/provenance.json +7 -0
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
package sessioningest
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"os"
|
|
5
|
+
"path/filepath"
|
|
6
|
+
"strings"
|
|
7
|
+
"testing"
|
|
8
|
+
"unicode/utf8"
|
|
9
|
+
|
|
10
|
+
sdk "github.com/MichaelKinsy/PiG/extensions/sdk"
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
// Cases for the modes the retrieval hints point at: turn, tools and stats.
|
|
14
|
+
|
|
15
|
+
func writeRichFixture(t *testing.T) string {
|
|
16
|
+
t.Helper()
|
|
17
|
+
path := filepath.Join(t.TempDir(), "rich.jsonl")
|
|
18
|
+
content := strings.Join([]string{
|
|
19
|
+
`{"type":"session","id":"session-rich-000000","version":"3","cwd":"/repo","timestamp":"2026-05-24T10:00:00.000Z"}`,
|
|
20
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:01.000Z","message":{"role":"user","content":[{"type":"text","text":"Fix the parser"}]}}`,
|
|
21
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:02.000Z","message":{"role":"assistant","provider":"acme","model":"m-1","content":[{"type":"thinking","thinking":"secret reasoning"},{"type":"text","text":"Reading first."},{"type":"toolCall","id":"c1","name":"read","arguments":{"path":"parser.go"}},{"type":"toolCall","id":"c2","name":"bash","arguments":{"command":"go test ./..."}}],"usage":{"input":100,"output":50,"cacheRead":10,"cacheWrite":5,"totalTokens":165,"cost":{"total":0.5}}}}`,
|
|
22
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:03.000Z","message":{"role":"toolResult","toolName":"read","toolCallId":"c1","content":[{"type":"text","text":"package parser"}]}}`,
|
|
23
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:04.000Z","message":{"role":"toolResult","toolName":"bash","toolCallId":"c2","isError":true,"content":[{"type":"text","text":"FAIL parser"}]}}`,
|
|
24
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:05.000Z","message":{"role":"assistant","provider":"acme","model":"m-1","content":[{"type":"toolCall","id":"c3","name":"edit","arguments":{"path":"parser.go","edits":[]}}],"usage":{"input":200,"output":20,"totalTokens":220,"cost":{"total":0.25}}}}`,
|
|
25
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:06.000Z","message":{"role":"toolResult","toolName":"edit","toolCallId":"c3","content":[{"type":"text","text":"edited"}]}}`,
|
|
26
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:07.000Z","message":{"role":"user","content":"Thanks"}}`,
|
|
27
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:10.000Z","message":{"role":"assistant","provider":"acme","model":"m-2","stopReason":"error","errorMessage":"rate limited","content":[]}}`,
|
|
28
|
+
`{"type":"model_change","provider":"acme","modelId":"m-2","timestamp":"2026-05-24T10:00:08.000Z"}`,
|
|
29
|
+
}, "\n") + "\n"
|
|
30
|
+
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
31
|
+
t.Fatal(err)
|
|
32
|
+
}
|
|
33
|
+
return path
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
func run(t *testing.T, params map[string]any) any {
|
|
37
|
+
t.Helper()
|
|
38
|
+
got, err := handleReadSession(sdk.Context{}, params)
|
|
39
|
+
if err != nil {
|
|
40
|
+
t.Fatalf("handleReadSession(%v) error = %v", params, err)
|
|
41
|
+
}
|
|
42
|
+
return got
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
func TestTurnModeJSONReturnsTheWholeTurn(t *testing.T) {
|
|
46
|
+
report, ok := run(t, map[string]any{"path": writeRichFixture(t), "mode": "turn", "turn": 1, "format": "json"}).(turnReport)
|
|
47
|
+
if !ok {
|
|
48
|
+
t.Fatal("result is not a turnReport")
|
|
49
|
+
}
|
|
50
|
+
if report.Turn != 1 || report.TotalTurns != 2 {
|
|
51
|
+
t.Fatalf("turn/total = %d/%d, want 1/2", report.Turn, report.TotalTurns)
|
|
52
|
+
}
|
|
53
|
+
roles := []string{}
|
|
54
|
+
for _, m := range report.Messages {
|
|
55
|
+
roles = append(roles, m.Role)
|
|
56
|
+
}
|
|
57
|
+
if strings.Join(roles, ",") != "user,assistant,toolResult,toolResult,assistant,toolResult" {
|
|
58
|
+
t.Fatalf("roles = %v", roles)
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
func TestTurnModeDefaultsAreTokenSafe(t *testing.T) {
|
|
63
|
+
res := run(t, map[string]any{"path": writeRichFixture(t), "mode": "turn", "turn": 1}).(sdk.ToolResult)
|
|
64
|
+
text := res.Content
|
|
65
|
+
for _, want := range []string{"# Session turn 1 of 2", "Fix the parser", "Reading first.", "read", "bash", "errors"} {
|
|
66
|
+
if !strings.Contains(text, want) {
|
|
67
|
+
t.Errorf("missing %q:\n%s", want, text)
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// Raw thinking and tool results are hidden unless asked for.
|
|
71
|
+
for _, hidden := range []string{"secret reasoning", "package parser", "FAIL parser"} {
|
|
72
|
+
if strings.Contains(text, hidden) {
|
|
73
|
+
t.Errorf("default output leaks %q:\n%s", hidden, text)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if res.Preview == "" {
|
|
77
|
+
t.Error("expected a Preview")
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
func TestTurnModeIncludeOptIns(t *testing.T) {
|
|
82
|
+
res := run(t, map[string]any{
|
|
83
|
+
"path": writeRichFixture(t), "mode": "turn", "turn": 1,
|
|
84
|
+
"include": []any{"text", "thinking", "tool_results", "usage"},
|
|
85
|
+
}).(sdk.ToolResult)
|
|
86
|
+
for _, want := range []string{"secret reasoning", "package parser", "FAIL parser", "$0.5000"} {
|
|
87
|
+
if !strings.Contains(res.Content, want) {
|
|
88
|
+
t.Errorf("missing %q:\n%s", want, res.Content)
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
func TestTurnModeHonorsMaxCharsPerItem(t *testing.T) {
|
|
94
|
+
res := run(t, map[string]any{
|
|
95
|
+
"path": writeRichFixture(t), "mode": "turn", "turn": 1, "include": []any{"text", "tool_results"}, "maxCharsPerItem": 4,
|
|
96
|
+
}).(sdk.ToolResult)
|
|
97
|
+
if strings.Contains(res.Content, "package parser") || !strings.Contains(res.Content, "truncated") {
|
|
98
|
+
t.Fatalf("expected truncated tool result:\n%s", res.Content)
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
func TestTurnModeValidatesTurn(t *testing.T) {
|
|
103
|
+
path := writeRichFixture(t)
|
|
104
|
+
for name, params := range map[string]map[string]any{
|
|
105
|
+
"missing": {"path": path, "mode": "turn"},
|
|
106
|
+
"zero": {"path": path, "mode": "turn", "turn": 0},
|
|
107
|
+
"past": {"path": path, "mode": "turn", "turn": 3},
|
|
108
|
+
} {
|
|
109
|
+
if _, err := handleReadSession(sdk.Context{}, params); err == nil || !strings.Contains(err.Error(), "turn") {
|
|
110
|
+
t.Errorf("%s: err = %v, want a turn error", name, err)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
func TestToolsModeCountsCallsResultsErrorsAndFiles(t *testing.T) {
|
|
116
|
+
report, ok := run(t, map[string]any{"path": writeRichFixture(t), "mode": "tools", "format": "json"}).(toolsReport)
|
|
117
|
+
if !ok {
|
|
118
|
+
t.Fatal("result is not a toolsReport")
|
|
119
|
+
}
|
|
120
|
+
byName := map[string]toolUsage{}
|
|
121
|
+
for _, u := range report.Tools {
|
|
122
|
+
byName[u.Name] = u
|
|
123
|
+
}
|
|
124
|
+
if u := byName["bash"]; u.Calls != 1 || u.Results != 1 || u.Errors != 1 {
|
|
125
|
+
t.Errorf("bash = %+v", u)
|
|
126
|
+
}
|
|
127
|
+
if u := byName["read"]; u.Calls != 1 || u.Results != 1 || u.Errors != 0 || u.FirstTurn != 1 {
|
|
128
|
+
t.Errorf("read = %+v", u)
|
|
129
|
+
}
|
|
130
|
+
if report.TotalCalls != 3 {
|
|
131
|
+
t.Errorf("TotalCalls = %d, want 3", report.TotalCalls)
|
|
132
|
+
}
|
|
133
|
+
files := map[string]string{}
|
|
134
|
+
for _, f := range report.Files {
|
|
135
|
+
files[f.Path] = strings.Join(f.Tools, "+")
|
|
136
|
+
}
|
|
137
|
+
if files["parser.go"] != "edit+read" {
|
|
138
|
+
t.Errorf("files = %v, want parser.go touched by edit+read", files)
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
func TestToolsModeMarkdownAndTurnFilter(t *testing.T) {
|
|
143
|
+
res := run(t, map[string]any{"path": writeRichFixture(t), "mode": "tools", "turn": 2}).(sdk.ToolResult)
|
|
144
|
+
if !strings.Contains(res.Content, "# Session tools") || !strings.Contains(res.Content, "No tool calls") {
|
|
145
|
+
t.Fatalf("turn 2 has no tools:\n%s", res.Content)
|
|
146
|
+
}
|
|
147
|
+
res = run(t, map[string]any{"path": writeRichFixture(t), "mode": "tools"}).(sdk.ToolResult)
|
|
148
|
+
for _, want := range []string{"| bash |", "| read |", "parser.go"} {
|
|
149
|
+
if !strings.Contains(res.Content, want) {
|
|
150
|
+
t.Errorf("missing %q:\n%s", want, res.Content)
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
func TestStatsModeTotals(t *testing.T) {
|
|
156
|
+
report, ok := run(t, map[string]any{"path": writeRichFixture(t), "mode": "stats", "format": "json"}).(statsReport)
|
|
157
|
+
if !ok {
|
|
158
|
+
t.Fatal("result is not a statsReport")
|
|
159
|
+
}
|
|
160
|
+
if report.Turns != 2 || report.Messages != 8 {
|
|
161
|
+
t.Errorf("turns/messages = %d/%d, want 2/8", report.Turns, report.Messages)
|
|
162
|
+
}
|
|
163
|
+
want := map[string]int{"user": 2, "assistant": 3, "toolResult": 3}
|
|
164
|
+
for role, n := range want {
|
|
165
|
+
if report.Roles[role] != n {
|
|
166
|
+
t.Errorf("roles[%s] = %d, want %d", role, report.Roles[role], n)
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
if report.Usage.Input != 300 || report.Usage.Output != 70 || report.Usage.Total != 385 || report.Usage.Cost != 0.75 {
|
|
170
|
+
t.Errorf("usage = %+v", report.Usage)
|
|
171
|
+
}
|
|
172
|
+
if report.ToolCalls != 3 || report.Errors != 2 {
|
|
173
|
+
t.Errorf("toolCalls/errors = %d/%d, want 3/2 (one tool error, one provider error)", report.ToolCalls, report.Errors)
|
|
174
|
+
}
|
|
175
|
+
models := map[string]int{}
|
|
176
|
+
for _, m := range report.Models {
|
|
177
|
+
models[m.Model] = m.Messages
|
|
178
|
+
}
|
|
179
|
+
if models["m-1"] != 2 || models["m-2"] != 1 {
|
|
180
|
+
t.Errorf("models = %v", models)
|
|
181
|
+
}
|
|
182
|
+
if report.DurationSeconds != 9 {
|
|
183
|
+
t.Errorf("duration = %d, want 9 (first to last message)", report.DurationSeconds)
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
func TestStatsModeMarkdown(t *testing.T) {
|
|
188
|
+
res := run(t, map[string]any{"path": writeRichFixture(t), "mode": "stats"}).(sdk.ToolResult)
|
|
189
|
+
for _, want := range []string{"# Session stats", "turns: 2", "tokens", "$0.7500", "m-1", "m-2"} {
|
|
190
|
+
if !strings.Contains(res.Content, want) {
|
|
191
|
+
t.Errorf("missing %q:\n%s", want, res.Content)
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
func TestSchemaHasNoUnusedFields(t *testing.T) {
|
|
197
|
+
props := readSessionSchema()["properties"].(map[string]any)
|
|
198
|
+
if _, ok := props["contextRadius"]; ok {
|
|
199
|
+
t.Error("contextRadius is not used by any mode and must not be in the schema")
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
func TestContinuationHintOnlyWhenThereIsMore(t *testing.T) {
|
|
204
|
+
path := writeRichFixture(t)
|
|
205
|
+
all := run(t, map[string]any{"path": path, "mode": "toc", "limit": 10}).(sdk.ToolResult).Content
|
|
206
|
+
if strings.Contains(all, "[Next:") {
|
|
207
|
+
t.Errorf("all turns fit but a Next hint was added:\n%s", all)
|
|
208
|
+
}
|
|
209
|
+
exact := run(t, map[string]any{"path": path, "mode": "toc", "limit": 2}).(sdk.ToolResult).Content
|
|
210
|
+
if strings.Contains(exact, "[Next:") {
|
|
211
|
+
t.Errorf("the page ends exactly at the last turn but a Next hint was added:\n%s", exact)
|
|
212
|
+
}
|
|
213
|
+
partial := run(t, map[string]any{"path": path, "mode": "toc", "limit": 1}).(sdk.ToolResult).Content
|
|
214
|
+
if !strings.Contains(partial, `"start":1`) {
|
|
215
|
+
t.Errorf("partial page needs a hint for start=1:\n%s", partial)
|
|
216
|
+
}
|
|
217
|
+
for _, mode := range []string{"stats"} {
|
|
218
|
+
out := run(t, map[string]any{"path": path, "mode": mode}).(sdk.ToolResult).Content
|
|
219
|
+
if strings.Contains(out, "[Next:") {
|
|
220
|
+
t.Errorf("mode %s has no pages but got a Next hint:\n%s", mode, out)
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
func TestTruncationNeverSplitsARune(t *testing.T) {
|
|
226
|
+
path := filepath.Join(t.TempDir(), "u.jsonl")
|
|
227
|
+
line := `{"type":"message","message":{"role":"user","content":"` + strings.Repeat("é", 50) + `"}}`
|
|
228
|
+
if err := os.WriteFile(path, []byte(line+"\n"), 0o644); err != nil {
|
|
229
|
+
t.Fatal(err)
|
|
230
|
+
}
|
|
231
|
+
res := run(t, map[string]any{"path": path, "mode": "turn", "turn": 1, "maxCharsPerItem": 11}).(sdk.ToolResult)
|
|
232
|
+
if !utf8.ValidString(res.Content) {
|
|
233
|
+
t.Fatalf("output is not valid UTF-8: %q", res.Content)
|
|
234
|
+
}
|
|
235
|
+
q := run(t, map[string]any{"path": path, "mode": "query", "query": "é", "maxCharsPerItem": 11, "format": "json"}).(queryReport)
|
|
236
|
+
if !utf8.ValidString(q.Hits[0].Excerpt) {
|
|
237
|
+
t.Fatalf("excerpt is not valid UTF-8: %q", q.Hits[0].Excerpt)
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
func TestOversizedTurnExplainsHowToNarrow(t *testing.T) {
|
|
242
|
+
path := filepath.Join(t.TempDir(), "big.jsonl")
|
|
243
|
+
line := `{"type":"message","message":{"role":"user","content":"` + strings.Repeat("x", 60*1024) + `"}}`
|
|
244
|
+
if err := os.WriteFile(path, []byte(line+"\n"), 0o644); err != nil {
|
|
245
|
+
t.Fatal(err)
|
|
246
|
+
}
|
|
247
|
+
out := run(t, map[string]any{"path": path, "mode": "turn", "turn": 1}).(sdk.ToolResult).Content
|
|
248
|
+
if !strings.Contains(out, "maxCharsPerItem") {
|
|
249
|
+
t.Errorf("truncated turn should say how to narrow it:\n%s", out[len(out)-300:])
|
|
250
|
+
}
|
|
251
|
+
if len(out) > 51*1024 {
|
|
252
|
+
t.Errorf("output is %d bytes, over the 50 KB cap", len(out))
|
|
253
|
+
}
|
|
254
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
package sessioningest
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"os"
|
|
5
|
+
"path/filepath"
|
|
6
|
+
"strings"
|
|
7
|
+
"testing"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
// Each line of the file used to be copied twice before it was decoded (Scanner.Text, then []byte of that string).
|
|
11
|
+
func TestParseSessionFileDoesNotCopyEachLineTwice(t *testing.T) {
|
|
12
|
+
var lines []string
|
|
13
|
+
lines = append(lines, `{"type":"session","id":"s1","version":3,"cwd":"/w","timestamp":"2026-01-01T00:00:00Z"}`)
|
|
14
|
+
pad := strings.Repeat("x", 4000)
|
|
15
|
+
for i := 0; i < 50; i++ {
|
|
16
|
+
lines = append(lines, `{"type":"message","id":"m","message":{"role":"toolResult","toolName":"bash","content":[{"type":"text","text":"`+pad+`"}]}}`)
|
|
17
|
+
}
|
|
18
|
+
path := filepath.Join(t.TempDir(), "s.jsonl")
|
|
19
|
+
if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
|
|
20
|
+
t.Fatal(err)
|
|
21
|
+
}
|
|
22
|
+
var bytes float64
|
|
23
|
+
allocBytes := testing.Benchmark(func(b *testing.B) {
|
|
24
|
+
b.ReportAllocs()
|
|
25
|
+
for i := 0; i < b.N; i++ {
|
|
26
|
+
if s, err := parseSessionFile(path, 100000); err != nil || len(s.Messages) != 50 {
|
|
27
|
+
b.Fatal(err, len(s.Messages))
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
})
|
|
31
|
+
bytes = float64(allocBytes.AllocedBytesPerOp())
|
|
32
|
+
// 50 lines of ~4.1 KB: the decoded text is kept (about 205 KB); a second copy of every line is not.
|
|
33
|
+
if bytes > 205_000*2.6 {
|
|
34
|
+
t.Fatalf("parsing allocated %.0f bytes for a 205 KB file; each line is being copied more than once", bytes)
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
package sessioningest
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"os"
|
|
5
|
+
"path/filepath"
|
|
6
|
+
"strings"
|
|
7
|
+
"testing"
|
|
8
|
+
"unicode/utf8"
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
// Cases added by the adversarial review of the move.
|
|
12
|
+
|
|
13
|
+
func writeOneUserMessage(t *testing.T, text string) string {
|
|
14
|
+
t.Helper()
|
|
15
|
+
path := filepath.Join(t.TempDir(), "one.jsonl")
|
|
16
|
+
line := `{"type":"message","message":{"role":"user","content":"` + text + `"}}`
|
|
17
|
+
if err := os.WriteFile(path, []byte(line+"\n"), 0o644); err != nil {
|
|
18
|
+
t.Fatal(err)
|
|
19
|
+
}
|
|
20
|
+
return path
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// maxCharsPerItem shortens what is shown. It must not shrink what is searched: a
|
|
24
|
+
// match past the first maxCharsPerItem characters is still a hit.
|
|
25
|
+
func TestQueryFindsMatchesPastMaxCharsPerItem(t *testing.T) {
|
|
26
|
+
path := writeOneUserMessage(t, strings.Repeat("a", 500)+" needle "+strings.Repeat("b", 500))
|
|
27
|
+
for _, caseSensitive := range []bool{false, true} {
|
|
28
|
+
q := run(t, map[string]any{"path": path, "mode": "query", "query": "needle", "caseSensitive": caseSensitive, "maxCharsPerItem": 100, "format": "json"}).(queryReport)
|
|
29
|
+
if q.TotalHits != 1 {
|
|
30
|
+
t.Fatalf("caseSensitive=%v: hits = %d, want 1 (the match is at character 501)", caseSensitive, q.TotalHits)
|
|
31
|
+
}
|
|
32
|
+
if !strings.Contains(q.Hits[0].Excerpt, "needle") {
|
|
33
|
+
t.Errorf("caseSensitive=%v: the excerpt must be centered on the match: %q", caseSensitive, q.Hits[0].Excerpt)
|
|
34
|
+
}
|
|
35
|
+
if n := utf8.RuneCountInString(q.Hits[0].Excerpt); n > 100+len(" ... [truncated, 9999 chars total]") {
|
|
36
|
+
t.Errorf("caseSensitive=%v: excerpt is %d characters, want it bounded by maxCharsPerItem", caseSensitive, n)
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Case folding can change byte lengths (U+0130 is 2 bytes, its lower case 3), so
|
|
42
|
+
// a match offset taken from strings.ToLower(text) is wrong in text itself.
|
|
43
|
+
func TestCaseInsensitiveExcerptUsesOffsetsOfTheOriginalText(t *testing.T) {
|
|
44
|
+
path := writeOneUserMessage(t, strings.Repeat("İ", 300)+" needle "+strings.Repeat("z", 300))
|
|
45
|
+
q := run(t, map[string]any{"path": path, "mode": "query", "query": "NEEDLE", "format": "json"}).(queryReport)
|
|
46
|
+
if q.TotalHits != 1 {
|
|
47
|
+
t.Fatalf("hits = %d, want 1", q.TotalHits)
|
|
48
|
+
}
|
|
49
|
+
excerpt := q.Hits[0].Excerpt
|
|
50
|
+
if !strings.Contains(excerpt, "needle") || !utf8.ValidString(excerpt) {
|
|
51
|
+
t.Fatalf("excerpt does not show the match, or splits a character: %q", excerpt)
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
package sessioningest
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"os"
|
|
5
|
+
"path/filepath"
|
|
6
|
+
"strings"
|
|
7
|
+
"testing"
|
|
8
|
+
|
|
9
|
+
sdk "github.com/MichaelKinsy/PiG/extensions/sdk"
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
func TestReadSessionTOCJSON(t *testing.T) {
|
|
13
|
+
path := writeSessionFixture(t)
|
|
14
|
+
got, err := handleReadSession(sdk.Context{}, map[string]any{
|
|
15
|
+
"path": path,
|
|
16
|
+
"mode": "toc",
|
|
17
|
+
"format": "json",
|
|
18
|
+
"limit": 10,
|
|
19
|
+
})
|
|
20
|
+
if err != nil {
|
|
21
|
+
t.Fatalf("handleReadSession() error = %v", err)
|
|
22
|
+
}
|
|
23
|
+
report, ok := got.(tocReport)
|
|
24
|
+
if !ok {
|
|
25
|
+
t.Fatalf("result type = %T, want tocReport", got)
|
|
26
|
+
}
|
|
27
|
+
if report.TotalTurns != 2 {
|
|
28
|
+
t.Fatalf("TotalTurns = %d, want 2", report.TotalTurns)
|
|
29
|
+
}
|
|
30
|
+
if report.Session.ID != "session-1234567890" {
|
|
31
|
+
t.Fatalf("session ID = %q", report.Session.ID)
|
|
32
|
+
}
|
|
33
|
+
if len(report.Items) != 2 {
|
|
34
|
+
t.Fatalf("items = %d, want 2", len(report.Items))
|
|
35
|
+
}
|
|
36
|
+
first := report.Items[0]
|
|
37
|
+
if first.TurnIndex != 1 || first.EntryIndex != 2 || first.MessageIndex != 1 {
|
|
38
|
+
t.Fatalf("first indices = %+v", first)
|
|
39
|
+
}
|
|
40
|
+
if first.User != "Please inspect the repo" {
|
|
41
|
+
t.Fatalf("first user = %q", first.User)
|
|
42
|
+
}
|
|
43
|
+
if strings.Join(first.Tools, ",") != "bash,read" {
|
|
44
|
+
t.Fatalf("first tools = %v, want bash/read", first.Tools)
|
|
45
|
+
}
|
|
46
|
+
if first.AssistantMessages != 1 || first.ToolResults != 1 || first.Errors != 1 {
|
|
47
|
+
t.Fatalf("first counts = %+v", first)
|
|
48
|
+
}
|
|
49
|
+
if first.Usage.Total != 30 || first.Usage.Cost != 0.12 {
|
|
50
|
+
t.Fatalf("first usage = %+v", first.Usage)
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
func TestReadSessionTOCMarkdown(t *testing.T) {
|
|
55
|
+
path := writeSessionFixture(t)
|
|
56
|
+
got, err := handleReadSession(sdk.Context{}, map[string]any{
|
|
57
|
+
"path": path,
|
|
58
|
+
"mode": "toc",
|
|
59
|
+
"format": "compact_markdown",
|
|
60
|
+
"limit": 1,
|
|
61
|
+
})
|
|
62
|
+
if err != nil {
|
|
63
|
+
t.Fatalf("handleReadSession() error = %v", err)
|
|
64
|
+
}
|
|
65
|
+
result, ok := got.(sdk.ToolResult)
|
|
66
|
+
if !ok {
|
|
67
|
+
t.Fatalf("result type = %T, want sdk.ToolResult", got)
|
|
68
|
+
}
|
|
69
|
+
text := result.Content
|
|
70
|
+
for _, want := range []string{"# Session TOC", "session-123", "| 1 |", "Please inspect the repo", "bash, read", "errors:1"} {
|
|
71
|
+
if !strings.Contains(text, want) {
|
|
72
|
+
t.Fatalf("markdown missing %q:\n%s", want, text)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if strings.Contains(text, "| 2 |") {
|
|
76
|
+
t.Fatalf("markdown included turn 2 despite limit=1:\n%s", text)
|
|
77
|
+
}
|
|
78
|
+
if result.Preview == "" {
|
|
79
|
+
t.Fatal("expected non-empty Preview")
|
|
80
|
+
}
|
|
81
|
+
if !strings.Contains(result.Preview, "2 turns") {
|
|
82
|
+
t.Errorf("preview should mention total turns: %q", result.Preview)
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
func TestReadSessionQueryJSON(t *testing.T) {
|
|
87
|
+
path := writeSessionFixture(t)
|
|
88
|
+
got, err := handleReadSession(sdk.Context{}, map[string]any{
|
|
89
|
+
"path": path,
|
|
90
|
+
"mode": "query",
|
|
91
|
+
"query": "inspect",
|
|
92
|
+
"format": "json",
|
|
93
|
+
"limit": 10,
|
|
94
|
+
})
|
|
95
|
+
if err != nil {
|
|
96
|
+
t.Fatalf("handleReadSession() error = %v", err)
|
|
97
|
+
}
|
|
98
|
+
report, ok := got.(queryReport)
|
|
99
|
+
if !ok {
|
|
100
|
+
t.Fatalf("result type = %T, want queryReport", got)
|
|
101
|
+
}
|
|
102
|
+
if report.TotalHits != 2 {
|
|
103
|
+
t.Fatalf("TotalHits = %d, want 2", report.TotalHits)
|
|
104
|
+
}
|
|
105
|
+
if len(report.Hits) != 2 {
|
|
106
|
+
t.Fatalf("hits = %d, want 2", len(report.Hits))
|
|
107
|
+
}
|
|
108
|
+
if report.Hits[0].Role != "user" || report.Hits[0].TurnIndex != 1 || !strings.Contains(report.Hits[0].Excerpt, "inspect") {
|
|
109
|
+
t.Fatalf("first hit = %+v, want user turn 1 inspect", report.Hits[0])
|
|
110
|
+
}
|
|
111
|
+
if report.Hits[1].Role != "assistant" || report.Hits[1].TurnIndex != 1 {
|
|
112
|
+
t.Fatalf("second hit = %+v, want assistant turn 1", report.Hits[1])
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
func TestReadSessionQueryFindsToolResultsAndHonorsCaseSensitivity(t *testing.T) {
|
|
117
|
+
path := writeSessionFixture(t)
|
|
118
|
+
got, err := handleReadSession(sdk.Context{}, map[string]any{
|
|
119
|
+
"path": path,
|
|
120
|
+
"mode": "query",
|
|
121
|
+
"query": "BOOM",
|
|
122
|
+
"caseSensitive": false,
|
|
123
|
+
"format": "json",
|
|
124
|
+
})
|
|
125
|
+
if err != nil {
|
|
126
|
+
t.Fatalf("handleReadSession() error = %v", err)
|
|
127
|
+
}
|
|
128
|
+
report := got.(queryReport)
|
|
129
|
+
if report.TotalHits != 1 || report.Hits[0].Role != "toolResult" || report.Hits[0].ToolName != "bash" {
|
|
130
|
+
t.Fatalf("report = %+v, want bash toolResult hit", report)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
got, err = handleReadSession(sdk.Context{}, map[string]any{
|
|
134
|
+
"path": path,
|
|
135
|
+
"mode": "query",
|
|
136
|
+
"query": "BOOM",
|
|
137
|
+
"caseSensitive": true,
|
|
138
|
+
"format": "json",
|
|
139
|
+
})
|
|
140
|
+
if err != nil {
|
|
141
|
+
t.Fatalf("handleReadSession() error = %v", err)
|
|
142
|
+
}
|
|
143
|
+
report = got.(queryReport)
|
|
144
|
+
if report.TotalHits != 0 {
|
|
145
|
+
t.Fatalf("case-sensitive TotalHits = %d, want 0", report.TotalHits)
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
func TestReadSessionQueryMarkdown(t *testing.T) {
|
|
150
|
+
path := writeSessionFixture(t)
|
|
151
|
+
got, err := handleReadSession(sdk.Context{}, map[string]any{
|
|
152
|
+
"path": path,
|
|
153
|
+
"mode": "query",
|
|
154
|
+
"query": "Done",
|
|
155
|
+
"format": "compact_markdown",
|
|
156
|
+
})
|
|
157
|
+
if err != nil {
|
|
158
|
+
t.Fatalf("handleReadSession() error = %v", err)
|
|
159
|
+
}
|
|
160
|
+
result, ok := got.(sdk.ToolResult)
|
|
161
|
+
if !ok {
|
|
162
|
+
t.Fatalf("result type = %T, want sdk.ToolResult", got)
|
|
163
|
+
}
|
|
164
|
+
text := result.Content
|
|
165
|
+
for _, want := range []string{"# Session query", "Done", "| 1 | 2 | assistant", "mode=turn turn=2"} {
|
|
166
|
+
if !strings.Contains(text, want) {
|
|
167
|
+
t.Fatalf("markdown missing %q:\n%s", want, text)
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
if result.Preview == "" {
|
|
171
|
+
t.Fatal("expected non-empty Preview")
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
func TestReadSessionSliceJSONWithRoleFilter(t *testing.T) {
|
|
176
|
+
path := writeSessionFixture(t)
|
|
177
|
+
got, err := handleReadSession(sdk.Context{}, map[string]any{
|
|
178
|
+
"path": path,
|
|
179
|
+
"mode": "slice",
|
|
180
|
+
"format": "json",
|
|
181
|
+
"start": 0,
|
|
182
|
+
"limit": 2,
|
|
183
|
+
"roles": []any{"assistant"},
|
|
184
|
+
})
|
|
185
|
+
if err != nil {
|
|
186
|
+
t.Fatalf("handleReadSession() error = %v", err)
|
|
187
|
+
}
|
|
188
|
+
report, ok := got.(sliceReport)
|
|
189
|
+
if !ok {
|
|
190
|
+
t.Fatalf("result type = %T, want sliceReport", got)
|
|
191
|
+
}
|
|
192
|
+
if report.FilteredTotal != 2 || len(report.Messages) != 2 {
|
|
193
|
+
t.Fatalf("filtered/messages = %d/%d, want 2/2", report.FilteredTotal, len(report.Messages))
|
|
194
|
+
}
|
|
195
|
+
if report.Messages[0].Role != "assistant" || report.Messages[0].TurnIndex != 1 || len(report.Messages[0].ToolCalls) != 2 {
|
|
196
|
+
t.Fatalf("first slice message = %+v", report.Messages[0])
|
|
197
|
+
}
|
|
198
|
+
if report.Messages[1].TurnIndex != 2 || report.Messages[1].Text[0] != "Done." {
|
|
199
|
+
t.Fatalf("second slice message = %+v", report.Messages[1])
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
func TestReadSessionSliceMarkdown(t *testing.T) {
|
|
204
|
+
path := writeSessionFixture(t)
|
|
205
|
+
got, err := handleReadSession(sdk.Context{}, map[string]any{
|
|
206
|
+
"path": path,
|
|
207
|
+
"mode": "slice",
|
|
208
|
+
"format": "compact_markdown",
|
|
209
|
+
"start": 1,
|
|
210
|
+
"limit": 2,
|
|
211
|
+
})
|
|
212
|
+
if err != nil {
|
|
213
|
+
t.Fatalf("handleReadSession() error = %v", err)
|
|
214
|
+
}
|
|
215
|
+
result, ok := got.(sdk.ToolResult)
|
|
216
|
+
if !ok {
|
|
217
|
+
t.Fatalf("result type = %T, want sdk.ToolResult", got)
|
|
218
|
+
}
|
|
219
|
+
text := result.Content
|
|
220
|
+
for _, want := range []string{"# Session slice", "message 2 turn 1 assistant", "tool_calls: bash, read", "message 3 turn 1 toolResult"} {
|
|
221
|
+
if !strings.Contains(text, want) {
|
|
222
|
+
t.Fatalf("markdown missing %q:\n%s", want, text)
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if result.Preview == "" {
|
|
226
|
+
t.Fatal("expected non-empty Preview")
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
func TestReadSessionQueryRequiresQuery(t *testing.T) {
|
|
231
|
+
path := writeSessionFixture(t)
|
|
232
|
+
_, err := handleReadSession(sdk.Context{}, map[string]any{"path": path, "mode": "query"})
|
|
233
|
+
if err == nil || !strings.Contains(err.Error(), "query is required") {
|
|
234
|
+
t.Fatalf("err = %v, want query required", err)
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
func TestReadSessionRejectsUnknownMode(t *testing.T) {
|
|
239
|
+
_, err := handleReadSession(sdk.Context{}, map[string]any{"path": "x", "mode": "missing"})
|
|
240
|
+
if err == nil || !strings.Contains(err.Error(), "unknown mode") {
|
|
241
|
+
t.Fatalf("err = %v, want unknown mode", err)
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
func TestReadSessionReportsMissingPath(t *testing.T) {
|
|
246
|
+
_, err := handleReadSession(sdk.Context{}, map[string]any{"mode": "toc"})
|
|
247
|
+
if err == nil || !strings.Contains(err.Error(), "path is required") {
|
|
248
|
+
t.Fatalf("err = %v, want path required", err)
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
func writeSessionFixture(t *testing.T) string {
|
|
253
|
+
t.Helper()
|
|
254
|
+
path := filepath.Join(t.TempDir(), "session.jsonl")
|
|
255
|
+
content := strings.Join([]string{
|
|
256
|
+
`{"type":"session","id":"session-1234567890","version":"0.75.4","cwd":"/repo","timestamp":"2026-05-24T10:00:00.000Z"}`,
|
|
257
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:01.000Z","message":{"role":"user","content":[{"type":"text","text":"Please inspect the repo"}]}}`,
|
|
258
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:02.000Z","message":{"role":"assistant","content":[{"type":"text","text":"I will inspect it."},{"type":"toolCall","id":"call-1","name":"bash","arguments":{"command":"ls"}},{"type":"toolCall","id":"call-2","name":"read","arguments":{"path":"README.md"}}],"usage":{"input":10,"output":20,"totalTokens":30,"cost":{"total":0.12}}}}`,
|
|
259
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:03.000Z","message":{"role":"toolResult","toolName":"bash","isError":true,"content":[{"type":"text","text":"boom"}]}}`,
|
|
260
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:04.000Z","message":{"role":"user","content":[{"type":"text","text":"Continue"}]}}`,
|
|
261
|
+
`{"type":"message","timestamp":"2026-05-24T10:00:05.000Z","message":{"role":"assistant","content":[{"type":"text","text":"Done."}],"usage":{"input":1,"output":2,"totalTokens":3}}}`,
|
|
262
|
+
}, "\n") + "\n"
|
|
263
|
+
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
264
|
+
t.Fatal(err)
|
|
265
|
+
}
|
|
266
|
+
return path
|
|
267
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@pi-in-go/pigpen-session-ingest",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "read_session: a token-safe tool that lets the model query Pi and PiG JSONL session transcripts (contents, search, single turns, tool usage, stats)",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"pig-package",
|
|
7
|
+
"sessions",
|
|
8
|
+
"history",
|
|
9
|
+
"extension",
|
|
10
|
+
"pigpen"
|
|
11
|
+
],
|
|
12
|
+
"license": "MIT",
|
|
13
|
+
"author": "Michael Kinsy",
|
|
14
|
+
"repository": {
|
|
15
|
+
"type": "git",
|
|
16
|
+
"url": "git+https://github.com/MichaelKinsy/pigpen.git",
|
|
17
|
+
"directory": "components/session-ingest"
|
|
18
|
+
},
|
|
19
|
+
"homepage": "https://github.com/MichaelKinsy/pigpen/tree/main/components/session-ingest#readme",
|
|
20
|
+
"bugs": {
|
|
21
|
+
"url": "https://github.com/MichaelKinsy/pigpen/issues"
|
|
22
|
+
},
|
|
23
|
+
"publishConfig": {
|
|
24
|
+
"access": "public"
|
|
25
|
+
},
|
|
26
|
+
"files": [
|
|
27
|
+
"extensions",
|
|
28
|
+
"README.md",
|
|
29
|
+
"LICENSE",
|
|
30
|
+
"CREDITS.md",
|
|
31
|
+
"provenance.json"
|
|
32
|
+
],
|
|
33
|
+
"pi": {
|
|
34
|
+
"extensions": [
|
|
35
|
+
"extensions/session-ingest"
|
|
36
|
+
]
|
|
37
|
+
}
|
|
38
|
+
}
|