little_ghost 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +90 -84
- data/docs/guides/assemblies.md +412 -0
- data/docs/guides/code_mode.md +275 -0
- data/docs/guides/core_concepts.md +150 -239
- data/docs/guides/getting_started.md +125 -87
- data/docs/guides/integrations.md +217 -0
- data/docs/guides/models_and_providers.md +125 -0
- data/docs/guides/production.md +253 -0
- data/docs/guides/prompt_views.md +139 -0
- data/docs/guides/sandboxing.md +282 -0
- data/docs/guides/skills.md +141 -0
- data/docs/guides/structured_outputs_and_content.md +135 -0
- data/docs/guides/tools.md +329 -0
- data/lib/little_ghost/ag_ui/adapter.rb +5 -5
- data/lib/little_ghost/agent/delegation.rb +3 -3
- data/lib/little_ghost/agent/skills.rb +6 -1
- data/lib/little_ghost/agent/tool_loop.rb +6 -2
- data/lib/little_ghost/agent.rb +411 -214
- data/lib/little_ghost/agent_builder.rb +61 -22
- data/lib/little_ghost/{agent_interruptions.rb → agent_interjections.rb} +12 -12
- data/lib/little_ghost/agent_stream_source.rb +262 -0
- data/lib/little_ghost/artifact.rb +182 -0
- data/lib/little_ghost/artifacts/presentation_budget.rb +44 -0
- data/lib/little_ghost/artifacts/workspace_store.rb +370 -0
- data/lib/little_ghost/artifacts.rb +11 -0
- data/lib/little_ghost/assembly.rb +110 -30
- data/lib/little_ghost/assembly_builder.rb +59 -21
- data/lib/little_ghost/assembly_execution.rb +110 -6
- data/lib/little_ghost/code_mode/broker.rb +164 -0
- data/lib/little_ghost/code_mode/catalog.rb +54 -0
- data/lib/little_ghost/code_mode/engine.rb +58 -0
- data/lib/little_ghost/code_mode/javascript/catalog.rb +118 -0
- data/lib/little_ghost/code_mode/javascript/client.rb +550 -0
- data/lib/little_ghost/code_mode/javascript/host.rb +347 -0
- data/lib/little_ghost/code_mode/javascript/session.rb +579 -0
- data/lib/little_ghost/code_mode/javascript_engine.rb +172 -0
- data/lib/little_ghost/code_mode/protocol.rb +78 -0
- data/lib/little_ghost/code_mode/ruby/catalog.rb +80 -0
- data/lib/little_ghost/code_mode/ruby/host.rb +102 -0
- data/lib/little_ghost/code_mode/ruby/session.rb +508 -0
- data/lib/little_ghost/code_mode/ruby_engine.rb +86 -0
- data/lib/little_ghost/code_mode/runtime.rb +335 -0
- data/lib/little_ghost/code_mode/session.rb +61 -0
- data/lib/little_ghost/code_mode/types.rb +58 -0
- data/lib/little_ghost/code_mode.rb +53 -0
- data/lib/little_ghost/configuration.rb +366 -47
- data/lib/little_ghost/content.rb +24 -13
- data/lib/little_ghost/data_map.rb +209 -0
- data/lib/little_ghost/errors.rb +28 -9
- data/lib/little_ghost/execution.rb +33 -34
- data/lib/little_ghost/graph.rb +376 -197
- data/lib/little_ghost/mcp/client.rb +487 -88
- data/lib/little_ghost/mcp/toolset.rb +210 -0
- data/lib/little_ghost/mcp/types.rb +216 -0
- data/lib/little_ghost/mcp.rb +3 -0
- data/lib/little_ghost/message.rb +4 -4
- data/lib/little_ghost/model_capabilities.rb +9 -6
- data/lib/little_ghost/model_request.rb +0 -12
- data/lib/little_ghost/model_resolver.rb +15 -5
- data/lib/little_ghost/model_response.rb +3 -7
- data/lib/little_ghost/network/authorizer_server.rb +162 -0
- data/lib/little_ghost/network/certificate_authority.rb +98 -0
- data/lib/little_ghost/network/envoy_config.rb +362 -0
- data/lib/little_ghost/network/envoy_gateway.rb +409 -0
- data/lib/little_ghost/network/external_gateway.rb +68 -0
- data/lib/little_ghost/network.rb +96 -0
- data/lib/little_ghost/prompt_resolver.rb +9 -7
- data/lib/little_ghost/provider_registry.rb +3 -3
- data/lib/little_ghost/providers/anthropic.rb +8 -1
- data/lib/little_ghost/providers/gemini.rb +10 -1
- data/lib/little_ghost/providers/vertex_ai.rb +6 -1
- data/lib/little_ghost/run.rb +162 -54
- data/lib/little_ghost/run_context.rb +42 -20
- data/lib/little_ghost/run_result.rb +0 -7
- data/lib/little_ghost/runtime/hook.rb +8 -3
- data/lib/little_ghost/runtime/hooks/artifacts.rb +338 -0
- data/lib/little_ghost/runtime.rb +160 -52
- data/lib/little_ghost/sandbox/capabilities.rb +68 -0
- data/lib/little_ghost/sandbox/environment_policy.rb +41 -0
- data/lib/little_ghost/sandbox/filesystem.rb +377 -0
- data/lib/little_ghost/sandbox/isolated_backend.rb +163 -0
- data/lib/little_ghost/sandbox/limits.rb +48 -0
- data/lib/little_ghost/sandbox/mount.rb +129 -0
- data/lib/little_ghost/sandbox/network_policy.rb +108 -0
- data/lib/little_ghost/sandbox/policy.rb +88 -0
- data/lib/little_ghost/sandbox/process_runner.rb +103 -0
- data/lib/little_ghost/sandbox/process_session.rb +335 -0
- data/lib/little_ghost/sandbox/scope.rb +304 -0
- data/lib/little_ghost/sandbox.rb +158 -32
- data/lib/little_ghost/sandboxes/bubblewrap.rb +351 -0
- data/lib/little_ghost/sandboxes/native.rb +51 -0
- data/lib/little_ghost/sandboxes/seatbelt.rb +261 -0
- data/lib/little_ghost/sandboxes/unrestricted.rb +241 -0
- data/lib/little_ghost/session.rb +39 -26
- data/lib/little_ghost/session_store.rb +9 -5
- data/lib/little_ghost/session_stores/agent_core_memory.rb +64 -56
- data/lib/little_ghost/session_stores/filesystem.rb +261 -0
- data/lib/little_ghost/session_stores/memory.rb +7 -0
- data/lib/little_ghost/skills/catalog.rb +94 -14
- data/lib/little_ghost/skills/resource_root.rb +42 -0
- data/lib/little_ghost/skills/skill.rb +0 -3
- data/lib/little_ghost/stream_event.rb +8 -13
- data/lib/little_ghost/subagents/control_tool.rb +8 -0
- data/lib/little_ghost/subagents/manager.rb +53 -50
- data/lib/little_ghost/support/callbacks.rb +3 -1
- data/lib/little_ghost/support/content_capture.rb +3 -3
- data/lib/little_ghost/support/http_client.rb +2 -2
- data/lib/little_ghost/support/redactor.rb +1 -1
- data/lib/little_ghost/swarm.rb +13 -5
- data/lib/little_ghost/tool.rb +157 -64
- data/lib/little_ghost/tool_registry.rb +1 -1
- data/lib/little_ghost/tools/filesystem.rb +1 -1
- data/lib/little_ghost/tools/shell.rb +4 -3
- data/lib/little_ghost/tools/write_todos.rb +6 -1
- data/lib/little_ghost/tracing/open_telemetry.rb +1 -1
- data/lib/little_ghost/version.rb +1 -1
- data/lib/little_ghost/workflow.rb +30 -21
- data/lib/little_ghost/workspace.rb +222 -8
- data/lib/little_ghost.rb +40 -27
- metadata +104 -3
- data/lib/little_ghost/unrestricted_sandbox.rb +0 -306
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "digest"
|
|
4
4
|
require "json"
|
|
5
5
|
require "securerandom"
|
|
6
|
+
require_relative "../data_map"
|
|
6
7
|
require_relative "../session_store"
|
|
7
8
|
|
|
8
9
|
module LittleGhost
|
|
@@ -49,10 +50,15 @@ module LittleGhost
|
|
|
49
50
|
# but horizontally scaled applications need an external lock or unique
|
|
50
51
|
# active-run record. Commits use generation and checkpoint records so an
|
|
51
52
|
# incomplete write is never exposed as a successful snapshot.
|
|
53
|
+
#
|
|
54
|
+
# AgentCore checkpoints are versioned. This store reads supported historical
|
|
55
|
+
# versions without writing during #load, then writes the current checkpoint
|
|
56
|
+
# format when the session next appends or replaces its snapshot.
|
|
52
57
|
class AgentCoreMemory < SessionStore
|
|
53
58
|
MESSAGE_PREFIX = "little_ghost:message:v4:" # :nodoc:
|
|
54
59
|
MESSAGE_CHUNK_PREFIX = "little_ghost:message_chunk:v4:" # :nodoc:
|
|
55
|
-
CHECKPOINT_PREFIX = "little_ghost:checkpoint:
|
|
60
|
+
CHECKPOINT_PREFIX = "little_ghost:checkpoint:v5:" # :nodoc:
|
|
61
|
+
LEGACY_CHECKPOINT_PREFIX = "little_ghost:checkpoint:v4:" # :nodoc:
|
|
56
62
|
CONVERSATIONAL_TEXT_LIMIT = 100_000 # :nodoc:
|
|
57
63
|
MESSAGE_CHUNK_CONTENT_LIMIT = 90_000 # :nodoc:
|
|
58
64
|
EVENT_PAYLOAD_LIMIT = 100 # :nodoc:
|
|
@@ -63,7 +69,8 @@ module LittleGhost
|
|
|
63
69
|
SYMBOL_KEY_PREFIX = "little_ghost:symbol:" # :nodoc:
|
|
64
70
|
STRING_KEY_PREFIX = "little_ghost:string:" # :nodoc:
|
|
65
71
|
MESSAGE_EVENT_TYPE = "message_v4" # :nodoc:
|
|
66
|
-
CHECKPOINT_EVENT_TYPE = "
|
|
72
|
+
CHECKPOINT_EVENT_TYPE = "checkpoint_v5" # :nodoc:
|
|
73
|
+
LEGACY_CHECKPOINT_EVENT_TYPE = "checkpoint_v4" # :nodoc:
|
|
67
74
|
CONVERSATION_PROJECTION_EVENT_TYPE = "conversation_projection_v1" # :nodoc:
|
|
68
75
|
PROJECTION_METADATA_KEYS = %w[
|
|
69
76
|
little_ghost_parent_link
|
|
@@ -135,27 +142,39 @@ module LittleGhost
|
|
|
135
142
|
# +expected_count+ matches the latest remote generation.
|
|
136
143
|
def append(id, messages:, state:, metadata:, expected_count:, actor_id: nil)
|
|
137
144
|
messages = persistable_messages(messages)
|
|
145
|
+
state = DataMap.new(state).to_h
|
|
146
|
+
metadata = DataMap.new(metadata).to_h
|
|
138
147
|
actor = self.class.safe_id(required_actor_id(actor_id))
|
|
139
148
|
session = self.class.safe_id(id)
|
|
140
149
|
key = [actor, session]
|
|
141
150
|
synchronize_persistence(key) do
|
|
142
|
-
head, = latest_checkpoint(actor, session)
|
|
151
|
+
head, lineage = latest_checkpoint(actor, session)
|
|
143
152
|
persistence = head&.fetch(:checkpoint)
|
|
144
153
|
persisted_count = persistence&.fetch(:message_count, 0) || 0
|
|
145
154
|
unless persisted_count == expected_count
|
|
146
155
|
raise ProtocolError, "Session changed while it was being updated"
|
|
147
156
|
end
|
|
148
157
|
|
|
149
|
-
|
|
158
|
+
if head && head.fetch(:format) != CHECKPOINT_EVENT_TYPE
|
|
159
|
+
existing = messages_from(message_records_for(actor, session, lineage:), lineage:)
|
|
160
|
+
messages = [*existing, *messages].freeze
|
|
161
|
+
expected_count = 0
|
|
162
|
+
persistence = head.fetch(:checkpoint)
|
|
163
|
+
root = true
|
|
164
|
+
generation = SecureRandom.uuid
|
|
165
|
+
else
|
|
166
|
+
root = persistence.nil?
|
|
167
|
+
generation = persistence&.fetch(:generation) || SecureRandom.uuid
|
|
168
|
+
end
|
|
150
169
|
commit_id = SecureRandom.uuid
|
|
151
170
|
plan = plan_messages(messages, generation:, commit_id:, offset: expected_count)
|
|
152
171
|
checkpoint = build_checkpoint(
|
|
153
172
|
persistence:,
|
|
154
173
|
generation:,
|
|
155
174
|
commit_id:,
|
|
156
|
-
root
|
|
175
|
+
root:,
|
|
157
176
|
plan:,
|
|
158
|
-
message_count: expected_count + messages.length,
|
|
177
|
+
message_count: root ? messages.length : expected_count + messages.length,
|
|
159
178
|
state:,
|
|
160
179
|
metadata:
|
|
161
180
|
)
|
|
@@ -167,6 +186,8 @@ module LittleGhost
|
|
|
167
186
|
# Replaces the visible snapshot by committing a new remote generation.
|
|
168
187
|
def replace(id, messages:, state:, metadata:, actor_id: nil)
|
|
169
188
|
messages = persistable_messages(messages)
|
|
189
|
+
state = DataMap.new(state).to_h
|
|
190
|
+
metadata = DataMap.new(metadata).to_h
|
|
170
191
|
actor = self.class.safe_id(required_actor_id(actor_id))
|
|
171
192
|
session = self.class.safe_id(id)
|
|
172
193
|
key = [actor, session]
|
|
@@ -379,27 +400,29 @@ module LittleGhost
|
|
|
379
400
|
|
|
380
401
|
def checkpoint_entries(actor_id, session_id)
|
|
381
402
|
entries = []
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
403
|
+
[LEGACY_CHECKPOINT_EVENT_TYPE, CHECKPOINT_EVENT_TYPE].each do |format|
|
|
404
|
+
each_event(
|
|
405
|
+
actor_id,
|
|
406
|
+
session_id,
|
|
407
|
+
filter: metadata_filter(type: format),
|
|
408
|
+
event_limit: MAX_CHECKPOINT_EVENTS,
|
|
409
|
+
payload_limit: MAX_CHECKPOINT_EVENTS,
|
|
410
|
+
byte_limit: MAX_CHECKPOINT_READ_BYTES
|
|
411
|
+
) do |event|
|
|
412
|
+
data = event_session_data(event, format:)
|
|
413
|
+
raise ProtocolError, "AgentCore session checkpoint event is invalid" unless data
|
|
414
|
+
unless event.event_timestamp.is_a?(Time)
|
|
415
|
+
raise ProtocolError, "AgentCore session checkpoint timestamp is invalid"
|
|
416
|
+
end
|
|
395
417
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
418
|
+
checkpoint = normalize_checkpoint(data, format:)
|
|
419
|
+
unless event_metadata_value(event, GENERATION_METADATA_KEY) == checkpoint.fetch(:generation) &&
|
|
420
|
+
event_metadata_value(event, COMMIT_METADATA_KEY) == checkpoint.fetch(:commit_id)
|
|
421
|
+
raise ProtocolError, "AgentCore session checkpoint metadata is invalid"
|
|
422
|
+
end
|
|
401
423
|
|
|
402
|
-
|
|
424
|
+
entries << {checkpoint:, event_timestamp: event.event_timestamp, format:}.freeze
|
|
425
|
+
end
|
|
403
426
|
end
|
|
404
427
|
entries
|
|
405
428
|
end
|
|
@@ -666,12 +689,13 @@ module LittleGhost
|
|
|
666
689
|
expected.length.times.map { |sequence| indexed.fetch(sequence) }
|
|
667
690
|
end
|
|
668
691
|
|
|
669
|
-
def event_session_data(event)
|
|
692
|
+
def event_session_data(event, format:)
|
|
693
|
+
prefix = (format == LEGACY_CHECKPOINT_EVENT_TYPE) ? LEGACY_CHECKPOINT_PREFIX : CHECKPOINT_PREFIX
|
|
670
694
|
values = event_payloads(event).filter_map do |payload|
|
|
671
695
|
blob = payload.respond_to?(:blob) ? payload.blob : nil
|
|
672
|
-
next unless blob.is_a?(String) && blob.start_with?(
|
|
696
|
+
next unless blob.is_a?(String) && blob.start_with?(prefix)
|
|
673
697
|
|
|
674
|
-
checkpoint = deserialize_checkpoint(blob.delete_prefix(
|
|
698
|
+
checkpoint = deserialize_checkpoint(blob.delete_prefix(prefix), format:)
|
|
675
699
|
checkpoint if checkpoint.is_a?(Hash)
|
|
676
700
|
end
|
|
677
701
|
values.one? ? values.first : nil
|
|
@@ -854,37 +878,21 @@ module LittleGhost
|
|
|
854
878
|
end
|
|
855
879
|
|
|
856
880
|
def serialize_checkpoint(checkpoint)
|
|
857
|
-
|
|
858
|
-
"state" => encode_hash_keys(checkpoint.fetch("state")),
|
|
859
|
-
"metadata" => encode_hash_keys(checkpoint.fetch("metadata"))
|
|
860
|
-
)
|
|
861
|
-
JSON.generate(document)
|
|
881
|
+
JSON.generate(checkpoint)
|
|
862
882
|
end
|
|
863
883
|
|
|
864
|
-
def deserialize_checkpoint(value)
|
|
884
|
+
def deserialize_checkpoint(value, format: CHECKPOINT_EVENT_TYPE)
|
|
865
885
|
document = JSON.parse(value)
|
|
866
886
|
return document unless document.is_a?(Hash)
|
|
867
887
|
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
end
|
|
872
|
-
|
|
873
|
-
def encode_hash_keys(value)
|
|
874
|
-
case value
|
|
875
|
-
when Hash
|
|
876
|
-
value.to_h do |key, child|
|
|
877
|
-
prefix = key.is_a?(Symbol) ? SYMBOL_KEY_PREFIX : STRING_KEY_PREFIX
|
|
878
|
-
["#{prefix}#{key}", encode_hash_keys(child)]
|
|
879
|
-
end
|
|
880
|
-
when Array
|
|
881
|
-
value.map { |child| encode_hash_keys(child) }
|
|
882
|
-
else
|
|
883
|
-
value
|
|
888
|
+
if format == LEGACY_CHECKPOINT_EVENT_TYPE
|
|
889
|
+
document["state"] = decode_legacy_hash_keys(document.fetch("state", {}))
|
|
890
|
+
document["metadata"] = decode_legacy_hash_keys(document.fetch("metadata", {}))
|
|
884
891
|
end
|
|
892
|
+
document
|
|
885
893
|
end
|
|
886
894
|
|
|
887
|
-
def
|
|
895
|
+
def decode_legacy_hash_keys(value)
|
|
888
896
|
case value
|
|
889
897
|
when Hash
|
|
890
898
|
value.to_h do |key, child|
|
|
@@ -893,10 +901,10 @@ module LittleGhost
|
|
|
893
901
|
else
|
|
894
902
|
key.delete_prefix(STRING_KEY_PREFIX)
|
|
895
903
|
end
|
|
896
|
-
[decoded_key,
|
|
904
|
+
[decoded_key, decode_legacy_hash_keys(child)]
|
|
897
905
|
end
|
|
898
906
|
when Array
|
|
899
|
-
value.map { |child|
|
|
907
|
+
value.map { |child| decode_legacy_hash_keys(child) }
|
|
900
908
|
else
|
|
901
909
|
value
|
|
902
910
|
end
|
|
@@ -984,7 +992,7 @@ module LittleGhost
|
|
|
984
992
|
end
|
|
985
993
|
end
|
|
986
994
|
|
|
987
|
-
def normalize_checkpoint(value)
|
|
995
|
+
def normalize_checkpoint(value, format: CHECKPOINT_EVENT_TYPE)
|
|
988
996
|
if serialize_checkpoint(value.transform_keys(&:to_s)).bytesize > MAX_CHECKPOINT_SERIALIZED_BYTES
|
|
989
997
|
raise ProtocolError, "AgentCore session checkpoint exceeds the byte limit"
|
|
990
998
|
end
|
|
@@ -1002,8 +1010,8 @@ module LittleGhost
|
|
|
1002
1010
|
serialized_bytes = data.fetch("serialized_bytes")
|
|
1003
1011
|
event_count = data.fetch("event_count")
|
|
1004
1012
|
payload_count = data.fetch("payload_count")
|
|
1005
|
-
state = data.fetch("state", {})
|
|
1006
|
-
metadata = data.fetch("metadata", {})
|
|
1013
|
+
state = DataMap.new(data.fetch("state", {})).to_h
|
|
1014
|
+
metadata = DataMap.new(data.fetch("metadata", {})).to_h
|
|
1007
1015
|
valid = valid_identifier?(generation) && valid_identifier?(commit_id) &&
|
|
1008
1016
|
(parent_commit_id.nil? || valid_identifier?(parent_commit_id)) &&
|
|
1009
1017
|
generation_revision.is_a?(Integer) && generation_revision.between?(1, MAX_REVISION) &&
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
require "securerandom"
|
|
7
|
+
require_relative "../data_map"
|
|
8
|
+
require_relative "../session_store"
|
|
9
|
+
|
|
10
|
+
module LittleGhost
|
|
11
|
+
module SessionStores
|
|
12
|
+
# Filesystem preserves LittleGhost sessions across process restarts in an
|
|
13
|
+
# application-controlled directory. Use it for durable local development,
|
|
14
|
+
# a single-host service, or processes that share a suitable filesystem.
|
|
15
|
+
#
|
|
16
|
+
# store = LittleGhost::SessionStores::Filesystem.new(
|
|
17
|
+
# root: "/var/lib/customer_support/sessions"
|
|
18
|
+
# )
|
|
19
|
+
#
|
|
20
|
+
# Configure the resulting store through Configuration#session_store, or
|
|
21
|
+
# pass it directly when opening a Session. Calls for the same session wait
|
|
22
|
+
# for one writer, including when separate Ruby processes share the root.
|
|
23
|
+
#
|
|
24
|
+
# **Warning:** The root contains readable session data and is not encrypted.
|
|
25
|
+
# Its complete path must be application-controlled: anyone able to read it
|
|
26
|
+
# can read session data, and anyone able to replace it can alter sessions.
|
|
27
|
+
#
|
|
28
|
+
# Session data is stored as ordinary JSON with canonical String keys. A
|
|
29
|
+
# value outside that boundary raises ProtocolError without replacing the
|
|
30
|
+
# previous snapshot. Shared roots require filesystem support for file
|
|
31
|
+
# locking and atomic rename.
|
|
32
|
+
class Filesystem < SessionStore
|
|
33
|
+
FORMAT_VERSION = 1 # :nodoc:
|
|
34
|
+
|
|
35
|
+
# Creates a store rooted at +root+ and creates the directory when needed.
|
|
36
|
+
#
|
|
37
|
+
# +root+ must be a private, non-symlinked directory. The application owns
|
|
38
|
+
# the complete path and must not let an untrusted request choose it.
|
|
39
|
+
# Raises ArgumentError when the root does not meet those requirements.
|
|
40
|
+
def initialize(root:)
|
|
41
|
+
super()
|
|
42
|
+
@root = File.expand_path(String(root))
|
|
43
|
+
raise ArgumentError, "root must not be empty" if root.to_s.empty?
|
|
44
|
+
|
|
45
|
+
FileUtils.mkdir_p(@root, mode: 0o700)
|
|
46
|
+
validate_root!
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Returns the stored snapshot for +id+, or +nil+ before the first write.
|
|
50
|
+
#
|
|
51
|
+
# +actor_id+ must match the actor that created an existing session.
|
|
52
|
+
# Raises Error for an actor mismatch and ProtocolError for an invalid or
|
|
53
|
+
# unsafe persisted snapshot.
|
|
54
|
+
def load(id, actor_id: nil)
|
|
55
|
+
with_lock(id) do
|
|
56
|
+
snapshot = read_snapshot(id)
|
|
57
|
+
validate_actor!(snapshot, actor_id) if snapshot
|
|
58
|
+
snapshot
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Atomically appends sanitized +messages+ and returns the updated snapshot.
|
|
63
|
+
#
|
|
64
|
+
# +expected_count+ must match the stored history length. +state+ and
|
|
65
|
+
# +metadata+ must meet this store's JSON boundary. Raises ProtocolError
|
|
66
|
+
# when another writer changed the session or the snapshot cannot be read
|
|
67
|
+
# or written. Raises Error when +actor_id+ does not match the session.
|
|
68
|
+
def append(id, messages:, state:, metadata:, expected_count:, actor_id: nil)
|
|
69
|
+
messages = persistable_messages(messages)
|
|
70
|
+
state = canonical_map(state)
|
|
71
|
+
metadata = canonical_map(metadata)
|
|
72
|
+
with_lock(id) do
|
|
73
|
+
current = read_snapshot(id)
|
|
74
|
+
validate_actor!(current, actor_id) if current
|
|
75
|
+
current ||= empty_snapshot(actor_id)
|
|
76
|
+
unless current.fetch(:messages).length == expected_count
|
|
77
|
+
raise ProtocolError, "Session changed while it was being updated"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
snapshot = {
|
|
81
|
+
actor_digest: current.fetch(:actor_digest),
|
|
82
|
+
messages: [*current.fetch(:messages), *messages].freeze,
|
|
83
|
+
state:,
|
|
84
|
+
metadata:
|
|
85
|
+
}.freeze
|
|
86
|
+
write_snapshot(id, snapshot)
|
|
87
|
+
public_snapshot(snapshot)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Atomically replaces the complete persisted snapshot and returns it.
|
|
92
|
+
#
|
|
93
|
+
# +state+, +metadata+, and +actor_id+ follow the same requirements as
|
|
94
|
+
# #append.
|
|
95
|
+
def replace(id, messages:, state:, metadata:, actor_id: nil)
|
|
96
|
+
messages = persistable_messages(messages)
|
|
97
|
+
state = canonical_map(state)
|
|
98
|
+
metadata = canonical_map(metadata)
|
|
99
|
+
with_lock(id) do
|
|
100
|
+
current = read_snapshot(id)
|
|
101
|
+
validate_actor!(current, actor_id) if current
|
|
102
|
+
snapshot = {
|
|
103
|
+
actor_digest: current ? current.fetch(:actor_digest) : actor_digest(actor_id),
|
|
104
|
+
messages: messages.freeze,
|
|
105
|
+
state:,
|
|
106
|
+
metadata:
|
|
107
|
+
}.freeze
|
|
108
|
+
write_snapshot(id, snapshot)
|
|
109
|
+
public_snapshot(snapshot)
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
private
|
|
114
|
+
|
|
115
|
+
def empty_snapshot(actor_id)
|
|
116
|
+
{
|
|
117
|
+
actor_digest: actor_digest(actor_id),
|
|
118
|
+
messages: [].freeze,
|
|
119
|
+
state: {}.freeze,
|
|
120
|
+
metadata: {}.freeze
|
|
121
|
+
}.freeze
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def public_snapshot(snapshot)
|
|
125
|
+
snapshot.slice(:messages, :state, :metadata)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def validate_actor!(snapshot, actor_id)
|
|
129
|
+
return if snapshot.fetch(:actor_digest) == actor_digest(actor_id)
|
|
130
|
+
|
|
131
|
+
raise Error, "Session actor does not match"
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def with_lock(id)
|
|
135
|
+
path = lock_path(id)
|
|
136
|
+
validate_entry!(path) if File.exist?(path) || File.symlink?(path)
|
|
137
|
+
File.open(path, File::RDWR | File::CREAT, 0o600) do |lock|
|
|
138
|
+
lock.flock(File::LOCK_EX)
|
|
139
|
+
yield
|
|
140
|
+
ensure
|
|
141
|
+
lock.flock(File::LOCK_UN)
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def read_snapshot(id)
|
|
146
|
+
path = snapshot_path(id)
|
|
147
|
+
return unless File.exist?(path) || File.symlink?(path)
|
|
148
|
+
|
|
149
|
+
validate_entry!(path)
|
|
150
|
+
|
|
151
|
+
document = JSON.parse(File.binread(path))
|
|
152
|
+
validate_document!(document)
|
|
153
|
+
snapshot = document.fetch("snapshot")
|
|
154
|
+
validate_snapshot!(snapshot)
|
|
155
|
+
{
|
|
156
|
+
actor_digest: document.fetch("actor_digest"),
|
|
157
|
+
messages: Array(snapshot.fetch("messages")).map { |message| Message.coerce(message) }.freeze,
|
|
158
|
+
state: snapshot.fetch("state"),
|
|
159
|
+
metadata: snapshot.fetch("metadata")
|
|
160
|
+
}.freeze
|
|
161
|
+
rescue JSON::ParserError, KeyError, TypeError, ArgumentError, NoMethodError => error
|
|
162
|
+
raise ProtocolError, "Filesystem session snapshot is invalid: #{error.class}"
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def write_snapshot(id, snapshot)
|
|
166
|
+
serialized_snapshot = public_snapshot(snapshot).merge(
|
|
167
|
+
messages: snapshot.fetch(:messages).map(&:to_h)
|
|
168
|
+
)
|
|
169
|
+
document = {
|
|
170
|
+
"version" => FORMAT_VERSION,
|
|
171
|
+
"actor_digest" => snapshot.fetch(:actor_digest),
|
|
172
|
+
"snapshot" => serialized_snapshot
|
|
173
|
+
}
|
|
174
|
+
document["digest"] = Digest::SHA256.hexdigest(JSON.generate(document))
|
|
175
|
+
write_atomically(snapshot_path(id), JSON.generate(document))
|
|
176
|
+
rescue JSON::GeneratorError, TypeError => error
|
|
177
|
+
raise ProtocolError, "Filesystem session snapshot could not be serialized: #{error.class}"
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def validate_document!(document)
|
|
181
|
+
raise ArgumentError unless document.is_a?(Hash)
|
|
182
|
+
raise ArgumentError unless document.fetch("version") == FORMAT_VERSION
|
|
183
|
+
raise ArgumentError unless document.fetch("actor_digest").match?(/\A[0-9a-f]{64}\z/)
|
|
184
|
+
digest = document.fetch("digest")
|
|
185
|
+
raise ArgumentError unless digest.is_a?(String) && digest.match?(/\A[0-9a-f]{64}\z/)
|
|
186
|
+
|
|
187
|
+
expected = document.except("digest")
|
|
188
|
+
raise ArgumentError unless Digest::SHA256.hexdigest(JSON.generate(expected)) == digest
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def validate_snapshot!(snapshot)
|
|
192
|
+
raise ArgumentError unless snapshot.is_a?(Hash)
|
|
193
|
+
raise ArgumentError unless snapshot.fetch("messages").is_a?(Array)
|
|
194
|
+
DataMap.new(snapshot.fetch("state"))
|
|
195
|
+
DataMap.new(snapshot.fetch("metadata"))
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def canonical_map(value)
|
|
199
|
+
DataMap.new(value).to_h
|
|
200
|
+
rescue ArgumentError => error
|
|
201
|
+
raise ProtocolError, "Filesystem session data is invalid: #{error.message}"
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def write_atomically(path, content)
|
|
205
|
+
validate_entry!(path) if File.exist?(path) || File.symlink?(path)
|
|
206
|
+
temporary = "#{path}.#{Process.pid}.#{SecureRandom.hex(8)}.tmp"
|
|
207
|
+
File.open(temporary, "wb", 0o600) do |file|
|
|
208
|
+
file.write(content)
|
|
209
|
+
file.flush
|
|
210
|
+
file.fsync
|
|
211
|
+
end
|
|
212
|
+
File.rename(temporary, path)
|
|
213
|
+
sync_root
|
|
214
|
+
ensure
|
|
215
|
+
File.delete(temporary) if temporary && File.exist?(temporary)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def sync_root
|
|
219
|
+
File.open(@root, File::RDONLY) { |directory| directory.fsync }
|
|
220
|
+
rescue SystemCallError
|
|
221
|
+
nil
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def validate_root!
|
|
225
|
+
stat = File.lstat(@root)
|
|
226
|
+
unless stat.directory? && !stat.symlink? && private_mode?(stat)
|
|
227
|
+
raise ArgumentError, "root must be a private, non-symlinked directory"
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def validate_entry!(path)
|
|
232
|
+
stat = File.lstat(path)
|
|
233
|
+
unless stat.file? && !stat.symlink? && private_mode?(stat)
|
|
234
|
+
raise ProtocolError, "Filesystem session entry is unsafe"
|
|
235
|
+
end
|
|
236
|
+
rescue Errno::ENOENT
|
|
237
|
+
nil
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def private_mode?(stat)
|
|
241
|
+
(stat.mode & 0o077).zero?
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def snapshot_path(id)
|
|
245
|
+
File.join(@root, "#{session_digest(id)}.json")
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def lock_path(id)
|
|
249
|
+
File.join(@root, "#{session_digest(id)}.lock")
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def session_digest(id)
|
|
253
|
+
Digest::SHA256.hexdigest("little_ghost/filesystem_session/v1\0#{String(id)}")
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def actor_digest(actor_id)
|
|
257
|
+
Digest::SHA256.hexdigest(JSON.generate([actor_id.nil?, actor_id&.to_s]))
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
end
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../data_map"
|
|
4
|
+
require_relative "../session_store"
|
|
5
|
+
|
|
3
6
|
module LittleGhost
|
|
4
7
|
# Ready-to-use persistence implementations for LittleGhost conversations.
|
|
5
8
|
module SessionStores
|
|
@@ -32,6 +35,8 @@ module LittleGhost
|
|
|
32
35
|
# stored conversation, then returns the updated snapshot.
|
|
33
36
|
def append(id, messages:, state:, metadata:, expected_count:, actor_id: nil)
|
|
34
37
|
messages = persistable_messages(messages)
|
|
38
|
+
state = DataMap.new(state).to_h
|
|
39
|
+
metadata = DataMap.new(metadata).to_h
|
|
35
40
|
@records_mutex.synchronize do
|
|
36
41
|
key = id.to_s
|
|
37
42
|
record = @records[key]
|
|
@@ -56,6 +61,8 @@ module LittleGhost
|
|
|
56
61
|
# Replaces the complete in-memory snapshot with sanitized +messages+.
|
|
57
62
|
def replace(id, messages:, state:, metadata:, actor_id: nil)
|
|
58
63
|
messages = persistable_messages(messages)
|
|
64
|
+
state = DataMap.new(state).to_h
|
|
65
|
+
metadata = DataMap.new(metadata).to_h
|
|
59
66
|
@records_mutex.synchronize do
|
|
60
67
|
key = id.to_s
|
|
61
68
|
validate_actor!(@records[key], actor_id)
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "yaml"
|
|
4
4
|
require "erb"
|
|
5
|
-
|
|
5
|
+
require_relative "resource_root"
|
|
6
6
|
|
|
7
7
|
module LittleGhost
|
|
8
8
|
module Skills
|
|
@@ -20,12 +20,16 @@ module LittleGhost
|
|
|
20
20
|
# YAML are rejected or skipped before instructions reach a model. Optional
|
|
21
21
|
# resource listings are limited by count and depth.
|
|
22
22
|
#
|
|
23
|
-
# ===
|
|
23
|
+
# === Choosing skill sources
|
|
24
24
|
#
|
|
25
|
-
#
|
|
26
|
-
# The +allowed-tools+ field
|
|
27
|
-
#
|
|
28
|
-
#
|
|
25
|
+
# Skill files become model instructions, so keep configured roots under
|
|
26
|
+
# application control and non-user-writable. The +allowed-tools+ field tells
|
|
27
|
+
# the model what a skill expects; the Agent's Tool list and each Tool's
|
|
28
|
+
# application checks still decide what can run. For a <tt>workspace://</tt>
|
|
29
|
+
# resource root, the Catalog verifies the named read-only grant and rejects
|
|
30
|
+
# direct writable aliases it can identify. LittleGhost cannot identify every
|
|
31
|
+
# alias created by an outer container or mount namespace, so the application
|
|
32
|
+
# must not expose the same files through another writable bind mount.
|
|
29
33
|
class Catalog
|
|
30
34
|
include Enumerable
|
|
31
35
|
|
|
@@ -40,21 +44,29 @@ module LittleGhost
|
|
|
40
44
|
SAFE_NAME_PATTERN = /\A[a-zA-Z0-9_-]+\z/ # :nodoc:
|
|
41
45
|
|
|
42
46
|
# Loads valid skills immediately using the supplied safety limits.
|
|
47
|
+
# +resource_root+ may be an absolute process-visible path. A
|
|
48
|
+
# <tt>workspace://name</tt> reference also requires +workspace+ and
|
|
49
|
+
# +sandbox+; it must resolve to every configured skill root through a
|
|
50
|
+
# read-only file-tool grant.
|
|
43
51
|
def initialize(
|
|
44
52
|
paths:,
|
|
45
53
|
max_skills: DEFAULT_MAX_SKILLS,
|
|
46
54
|
max_file_bytes: DEFAULT_MAX_FILE_BYTES,
|
|
47
55
|
max_resource_files: DEFAULT_MAX_RESOURCE_FILES,
|
|
48
56
|
only: nil,
|
|
49
|
-
resource_root: nil
|
|
57
|
+
resource_root: nil,
|
|
58
|
+
workspace: nil,
|
|
59
|
+
sandbox: nil
|
|
50
60
|
)
|
|
51
61
|
@paths = PathSet.new(paths)
|
|
52
62
|
@max_skills = positive_integer(max_skills, :max_skills)
|
|
53
63
|
@max_file_bytes = positive_integer(max_file_bytes, :max_file_bytes)
|
|
54
64
|
@max_resource_files = positive_integer(max_resource_files, :max_resource_files)
|
|
55
65
|
@only = Array(only).map(&:to_s).freeze if only
|
|
56
|
-
@resource_root =
|
|
66
|
+
@resource_root = ResourceRoot.normalize(resource_root)
|
|
67
|
+
validate_workspace_resource_root!(workspace, sandbox)
|
|
57
68
|
@skills = load_skills
|
|
69
|
+
validate_workspace_resource_aliases!(sandbox)
|
|
58
70
|
end
|
|
59
71
|
|
|
60
72
|
# Yields each Skill in lookup order.
|
|
@@ -257,15 +269,83 @@ module LittleGhost
|
|
|
257
269
|
integer
|
|
258
270
|
end
|
|
259
271
|
|
|
260
|
-
def
|
|
261
|
-
return unless
|
|
272
|
+
def validate_workspace_resource_root!(workspace, sandbox)
|
|
273
|
+
return unless @resource_root&.start_with?("workspace://")
|
|
274
|
+
unless workspace && sandbox
|
|
275
|
+
raise ConfigurationError, "workspace resource_root requires a Workspace and Sandbox"
|
|
276
|
+
end
|
|
277
|
+
unless sandbox.workspace.equal?(workspace)
|
|
278
|
+
raise ConfigurationError, "workspace resource_root requires the Sandbox bound to its Workspace"
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
physical_root = File.realpath(workspace.resolve(@resource_root))
|
|
282
|
+
source_roots = @paths.filter_map do |root|
|
|
283
|
+
File.realpath(root.path) if Dir.exist?(root.path)
|
|
284
|
+
end
|
|
285
|
+
unless source_roots.all? { |source_root| source_root == physical_root }
|
|
286
|
+
raise ConfigurationError, "workspace resource_root must map to each skill path"
|
|
287
|
+
end
|
|
288
|
+
unless sandbox.allows?(:filesystem_read, @resource_root) &&
|
|
289
|
+
!sandbox.allows?(:filesystem_write, @resource_root)
|
|
290
|
+
raise ConfigurationError, "workspace resource_root must be tool-readable and read-only"
|
|
291
|
+
end
|
|
292
|
+
if writable_tool_grants(sandbox).any? do |grant|
|
|
293
|
+
source = File.realpath(grant.source)
|
|
294
|
+
source == physical_root || source.start_with?("#{physical_root}#{File::SEPARATOR}")
|
|
295
|
+
end
|
|
296
|
+
raise ConfigurationError, "workspace resource_root must not contain writable file grants"
|
|
297
|
+
end
|
|
298
|
+
@workspace_physical_root = physical_root
|
|
299
|
+
rescue Errno::ENOENT, KeyError, ArgumentError => error
|
|
300
|
+
raise ConfigurationError, "workspace resource_root is not available: #{error.message}"
|
|
301
|
+
end
|
|
262
302
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
303
|
+
def validate_workspace_resource_aliases!(sandbox)
|
|
304
|
+
return unless @workspace_physical_root
|
|
305
|
+
|
|
306
|
+
protected_identities = protected_directory_identities
|
|
307
|
+
aliased = writable_tool_grants(sandbox).any? do |grant|
|
|
308
|
+
stat = File.stat(grant.source)
|
|
309
|
+
protected_identities.include?([stat.dev, stat.ino])
|
|
310
|
+
end
|
|
311
|
+
if aliased
|
|
312
|
+
raise ConfigurationError, "workspace resource_root must not contain writable file grants"
|
|
266
313
|
end
|
|
314
|
+
rescue Errno::ENOENT, Errno::EACCES => error
|
|
315
|
+
raise ConfigurationError, "workspace resource_root is not available: #{error.message}"
|
|
316
|
+
end
|
|
267
317
|
|
|
268
|
-
|
|
318
|
+
def writable_tool_grants(sandbox)
|
|
319
|
+
grants = sandbox.respond_to?(:process_grants) ? sandbox.process_grants : sandbox.scope.process_grants
|
|
320
|
+
grants.select { |grant| grant.tool_visible? && grant.writable? }
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
def protected_directory_identities
|
|
324
|
+
directories = [@workspace_physical_root]
|
|
325
|
+
@skills.each_value do |skill|
|
|
326
|
+
skill_directory = File.dirname(skill.source_path)
|
|
327
|
+
directories << skill_directory
|
|
328
|
+
RESOURCE_DIRECTORIES.each do |name|
|
|
329
|
+
resource_root = File.join(skill_directory, name)
|
|
330
|
+
collect_resource_directories(resource_root, directories) if File.directory?(resource_root)
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
directories.uniq.to_h do |directory|
|
|
334
|
+
stat = File.stat(directory)
|
|
335
|
+
[[stat.dev, stat.ino], true]
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
def collect_resource_directories(directory, directories, depth = 0)
|
|
340
|
+
return if depth >= MAX_RESOURCE_DEPTH || File.symlink?(directory)
|
|
341
|
+
|
|
342
|
+
directories << directory
|
|
343
|
+
Dir.children(directory).each do |name|
|
|
344
|
+
child = File.join(directory, name)
|
|
345
|
+
collect_resource_directories(child, directories, depth + 1) if File.directory?(child)
|
|
346
|
+
rescue Errno::ENOENT, Errno::EACCES
|
|
347
|
+
next
|
|
348
|
+
end
|
|
269
349
|
end
|
|
270
350
|
|
|
271
351
|
def agent_path(source_path, source_root)
|