rubik 0.8.1 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ATTRIBUTION.md +2 -2
- data/CHANGELOG.md +207 -43
- data/README.md +5 -1
- data/bin/rubik +15 -10
- data/lib/rubik/adapter/base.rb +19 -0
- data/lib/rubik/adapter/generic.rb +0 -14
- data/lib/rubik/adapter/node.rb +2 -0
- data/lib/rubik/ast/treesitter/zig_walker.rb +15 -2
- data/lib/rubik/cli/workspace_trust.rb +17 -0
- data/lib/rubik/concurrency/worker.rb +143 -0
- data/lib/rubik/concurrency.rb +1 -0
- data/lib/rubik/config/deep_ops.rb +17 -0
- data/lib/rubik/config/defaults.rb +23 -1
- data/lib/rubik/config/schema.rb +0 -1
- data/lib/rubik/config/store.rb +14 -0
- data/lib/rubik/config.rb +7 -3
- data/lib/rubik/context/repo_context_reader.rb +27 -8
- data/lib/rubik/events.rb +18 -0
- data/lib/rubik/kernel/dispatcher.rb +4 -4
- data/lib/rubik/kernel/unit.rb +4 -3
- data/lib/rubik/log.rb +2 -1
- data/lib/rubik/lsp/client.rb +83 -180
- data/lib/rubik/lsp/document_store.rb +209 -0
- data/lib/rubik/mcp/client.rb +4 -3
- data/lib/rubik/mcp/manager.rb +6 -2
- data/lib/rubik/mcp/transport/stdio.rb +51 -9
- data/lib/rubik/orchestrator/auto_apply.rb +2 -0
- data/lib/rubik/orchestrator/budget_manager.rb +18 -2
- data/lib/rubik/orchestrator/goal_ledger/deep_ops.rb +41 -0
- data/lib/rubik/orchestrator/goal_ledger/drift.rb +2 -1
- data/lib/rubik/orchestrator/goal_ledger/ledger.rb +56 -50
- data/lib/rubik/orchestrator/goal_ledger/observation_projection.rb +26 -0
- data/lib/rubik/orchestrator/goal_ledger/patch.rb +10 -17
- data/lib/rubik/orchestrator/goal_ledger/suspend_lifecycle.rb +9 -41
- data/lib/rubik/orchestrator/goal_ledger/validator.rb +6 -20
- data/lib/rubik/orchestrator/goal_ledger/verifier.rb +3 -14
- data/lib/rubik/orchestrator/goal_ledger/xml_render.rb +21 -14
- data/lib/rubik/orchestrator/goal_session.rb +505 -0
- data/lib/rubik/orchestrator/goal_state.rb +15 -15
- data/lib/rubik/orchestrator/loop.rb +62 -63
- data/lib/rubik/orchestrator/lsp_feedback.rb +7 -1
- data/lib/rubik/orchestrator/subagent/admission_policy.rb +161 -0
- data/lib/rubik/orchestrator/subagent/builtin_types.rb +3 -2
- data/lib/rubik/orchestrator/subagent/instance.rb +4 -0
- data/lib/rubik/orchestrator/subagent/journal.rb +2 -1
- data/lib/rubik/orchestrator/subagent/notices.rb +5 -4
- data/lib/rubik/orchestrator/subagent/profile_contract.rb +4 -10
- data/lib/rubik/orchestrator/subagent/questioner.rb +21 -6
- data/lib/rubik/orchestrator/subagent/run_cycle.rb +22 -1
- data/lib/rubik/orchestrator/subagent/scoped_registry.rb +34 -2
- data/lib/rubik/orchestrator/subagent/step_runner.rb +1 -1
- data/lib/rubik/orchestrator/subagent/teardown.rb +4 -0
- data/lib/rubik/orchestrator/subagent/transcript_formatter.rb +2 -1
- data/lib/rubik/orchestrator/subagent/type_registry.rb +19 -1
- data/lib/rubik/orchestrator/turn_executor/subagent_channel.rb +6 -1
- data/lib/rubik/orchestrator/turn_executor/user_channel.rb +6 -1
- data/lib/rubik/orchestrator/turn_executor.rb +2 -1
- data/lib/rubik/orchestrator.rb +1 -0
- data/lib/rubik/platform/path.rb +0 -1
- data/lib/rubik/platform.rb +15 -0
- data/lib/rubik/plugin/storage.rb +20 -29
- data/lib/rubik/prompt/context_builder.rb +24 -18
- data/lib/rubik/prompt/context_compactor.rb +31 -8
- data/lib/rubik/prompt/conversation_rows.rb +33 -0
- data/lib/rubik/prompt/doctrine.rb +4 -2
- data/lib/rubik/prompt/emergency_compactor.rb +5 -4
- data/lib/rubik/repl/commands/agent_context.rb +1 -1
- data/lib/rubik/repl/commands/agent_execution.rb +15 -467
- data/lib/rubik/repl/commands/buffer.rb +5 -1
- data/lib/rubik/repl/commands/lsp.rb +1 -1
- data/lib/rubik/repl/commands/mcp.rb +5 -0
- data/lib/rubik/repl/commands/subagents.rb +4 -4
- data/lib/rubik/repl/commands/workspace.rb +22 -2
- data/lib/rubik/repl/config_wiring.rb +24 -7
- data/lib/rubik/repl/input_execution.rb +1 -1
- data/lib/rubik/repl/input_loop.rb +1 -1
- data/lib/rubik/repl/lifecycle.rb +31 -2
- data/lib/rubik/repl/pruner/harness.rb +52 -67
- data/lib/rubik/repl/pruner/rules.rb +3 -1
- data/lib/rubik/repl/service_catalog.rb +1 -1
- data/lib/rubik/repl/synthesized_command_context.rb +3 -0
- data/lib/rubik/repl/turn_recorder.rb +5 -4
- data/lib/rubik/repl.rb +1 -0
- data/lib/rubik/runtime/context.rb +5 -0
- data/lib/rubik/runtime/discovery.rb +55 -0
- data/lib/rubik/runtime/manifest_store.rb +116 -0
- data/lib/rubik/runtime/overlay_storage.rb +300 -218
- data/lib/rubik/runtime/repair_probe.rb +20 -2
- data/lib/rubik/runtime/sealed_coeffects.rb +58 -0
- data/lib/rubik/runtime/synthesized_command_context.rb +80 -0
- data/lib/rubik/runtime/synthesized_tool_context.rb +123 -0
- data/lib/rubik/runtime/task_manager.rb +7 -2
- data/lib/rubik/runtime.rb +3 -0
- data/lib/rubik/security/allowlist.rb +167 -0
- data/lib/rubik/security/audit_log.rb +72 -0
- data/lib/rubik/security/classifier/classifier.rb +24 -8
- data/lib/rubik/security/classifier/commands/biome.yaml +49 -0
- data/lib/rubik/security/classifier/commands/bun.yaml +91 -0
- data/lib/rubik/security/classifier/commands/bundle.yaml +107 -0
- data/lib/rubik/security/classifier/commands/clang-format.yaml +21 -0
- data/lib/rubik/security/classifier/commands/clang.yaml +37 -0
- data/lib/rubik/security/classifier/commands/cmake.yaml +37 -0
- data/lib/rubik/security/classifier/commands/ctest.yaml +30 -0
- data/lib/rubik/security/classifier/commands/gcc.yaml +36 -0
- data/lib/rubik/security/classifier/commands/gem.yaml +104 -0
- data/lib/rubik/security/classifier/commands/gofmt.yaml +15 -0
- data/lib/rubik/security/classifier/commands/pnpm.yaml +8 -8
- data/lib/rubik/security/classifier/commands/python.yaml +18 -1
- data/lib/rubik/security/classifier/commands/python3.yaml +18 -1
- data/lib/rubik/security/classifier/commands/rake.yaml +60 -0
- data/lib/rubik/security/classifier/commands/rspec.yaml +42 -0
- data/lib/rubik/security/classifier/commands/rubocop.yaml +38 -0
- data/lib/rubik/security/classifier/commands/ruby.yaml +25 -0
- data/lib/rubik/security/classifier/commands/rustc.yaml +34 -0
- data/lib/rubik/security/classifier/commands/zig.yaml +77 -0
- data/lib/rubik/security/classifier/database.rb +12 -1
- data/lib/rubik/security/classifier/matcher.rb +17 -2
- data/lib/rubik/security/classifier/models.rb +3 -2
- data/lib/rubik/security/classifier.rb +5 -0
- data/lib/rubik/security/command_gate.rb +567 -0
- data/lib/rubik/security/guard.rb +96 -989
- data/lib/rubik/security/path_policy.rb +117 -0
- data/lib/rubik/security/synthesis_gate.rb +300 -0
- data/lib/rubik/security.rb +5 -0
- data/lib/rubik/storage/ranking.rb +113 -0
- data/lib/rubik/storage/store.rb +2 -92
- data/lib/rubik/storage/transcript_indexer.rb +2 -3
- data/lib/rubik/storage/vakanuvis/harness.rb +49 -89
- data/lib/rubik/tools/base.rb +24 -7
- data/lib/rubik/tools/builtins/files.rb +2 -0
- data/lib/rubik/tools/builtins/goal_ledger.rb +9 -9
- data/lib/rubik/tools/builtins/grep.rb +2 -3
- data/lib/rubik/tools/builtins/lifecycle.rb +91 -88
- data/lib/rubik/tools/builtins/synthesis.rb +124 -64
- data/lib/rubik/tools/builtins/unsynthesize.rb +38 -0
- data/lib/rubik/tools/catalog.rb +3 -0
- data/lib/rubik/tools/declaration.rb +6 -2
- data/lib/rubik/tools/lifecycle_rollbacks.rb +87 -0
- data/lib/rubik/tools/native_search.rb +2 -1
- data/lib/rubik/tools/preflight_runner.rb +107 -14
- data/lib/rubik/tools/registry.rb +2 -0
- data/lib/rubik/tools/truncator.rb +25 -0
- data/lib/rubik/tools/uri_resolver.rb +3 -1
- data/lib/rubik/uris.rb +5 -1
- data/lib/rubik/version.rb +1 -1
- data/lib/rubik.rb +1 -0
- metadata +35 -1
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../platform"
|
|
4
|
+
require_relative "errors"
|
|
5
|
+
|
|
6
|
+
module Rubik
|
|
7
|
+
module Security
|
|
8
|
+
# Critical-path write policy, split out of Guard (#403): the default
|
|
9
|
+
# critical patterns, the path-write verifier, the workspace-root
|
|
10
|
+
# verifier, and the command write-target walk. Receives the shared
|
|
11
|
+
# AuditLog instance — every block decision is audit-logged there.
|
|
12
|
+
class PathPolicy
|
|
13
|
+
DEFAULT_CRITICAL_PATTERNS = [
|
|
14
|
+
".git/**",
|
|
15
|
+
".rubik/**",
|
|
16
|
+
"**/.rubik/**",
|
|
17
|
+
".env*",
|
|
18
|
+
"secrets.*",
|
|
19
|
+
"*.pem",
|
|
20
|
+
"*.key",
|
|
21
|
+
"id_rsa*"
|
|
22
|
+
].freeze
|
|
23
|
+
|
|
24
|
+
def initialize(audit:, custom_patterns: [])
|
|
25
|
+
@critical_patterns = (DEFAULT_CRITICAL_PATTERNS + custom_patterns).uniq
|
|
26
|
+
@audit = audit
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def verify_path_write!(path)
|
|
30
|
+
return true if path.nil? || path.to_s.strip.empty?
|
|
31
|
+
|
|
32
|
+
raw = Rubik::Platform.normalize_path_raw(path.to_s)
|
|
33
|
+
expanded = Rubik::Platform.normalize_path_raw(File.expand_path(path.to_s))
|
|
34
|
+
|
|
35
|
+
# Check raw relative path, expanded path, and relative-from-pwd
|
|
36
|
+
candidates = [raw, expanded]
|
|
37
|
+
begin
|
|
38
|
+
pwd_raw = Rubik::Platform.normalize_path_raw(Dir.pwd)
|
|
39
|
+
candidates << expanded.delete_prefix("#{pwd_raw}/") if expanded.start_with?("#{pwd_raw}/")
|
|
40
|
+
rescue SystemCallError, ArgumentError
|
|
41
|
+
nil
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
candidates.uniq.each do |c|
|
|
45
|
+
if critical_path?(c)
|
|
46
|
+
@audit.log_audit("blocked_write", path: path.to_s, reason: "critical_path")
|
|
47
|
+
raise BlockedWriteError, "Write blocked on critical path: #{path}"
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Symlink re-check (fail-closed): the lexical candidates above cannot
|
|
52
|
+
# see `foo -> .git/config`. Resolve the deepest EXISTING ancestor and
|
|
53
|
+
# re-check; a missing tail is reattached lexically so ordinary
|
|
54
|
+
# new-file writes resolve to their lexical equivalent. A nil here
|
|
55
|
+
# means containment is unprovable (symlink loop / permission wall).
|
|
56
|
+
resolved = Rubik::Platform.resolved_path(path.to_s, strict: true)
|
|
57
|
+
if resolved.nil?
|
|
58
|
+
@audit.log_audit("blocked_write", path: path.to_s, reason: "path_resolution_failed")
|
|
59
|
+
raise BlockedWriteError, "Write blocked: path could not be safely resolved: #{path}"
|
|
60
|
+
end
|
|
61
|
+
if critical_path?(resolved)
|
|
62
|
+
@audit.log_audit("blocked_write", path: path.to_s, reason: "critical_path_symlink", resolved: resolved)
|
|
63
|
+
raise BlockedWriteError, "Write blocked on critical path (symlink-resolved): #{path} -> #{resolved}"
|
|
64
|
+
end
|
|
65
|
+
true
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def verify_root_safety!(path)
|
|
69
|
+
return true if path.nil? || path.to_s.strip.empty?
|
|
70
|
+
|
|
71
|
+
expanded = File.expand_path(path.to_s)
|
|
72
|
+
normalized = Rubik::Platform.normalize_path(expanded)
|
|
73
|
+
dangerous_roots = Rubik::Platform.dangerous_system_roots
|
|
74
|
+
|
|
75
|
+
if dangerous_roots.include?(normalized) ||
|
|
76
|
+
Rubik::Platform.drive_root?(expanded) ||
|
|
77
|
+
Rubik::Platform.path_equal?(expanded, Rubik::Platform.home_dir)
|
|
78
|
+
@audit.log_audit("blocked_root_addition", path: expanded, reason: "dangerous_system_root")
|
|
79
|
+
raise BlockedWriteError, "Adding dangerous system root as workspace is blocked: #{path} (#{expanded})"
|
|
80
|
+
end
|
|
81
|
+
true
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def critical_path?(path)
|
|
85
|
+
norm_path = path.to_s.tr("\\", "/").gsub(%r{^\./}, "")
|
|
86
|
+
@critical_patterns.any? do |pattern|
|
|
87
|
+
File.fnmatch?(pattern, norm_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) ||
|
|
88
|
+
File.fnmatch?("**/#{pattern}", norm_path, File::FNM_PATHNAME | File::FNM_DOTMATCH)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def check_critical_path_writes!(ast, cmd_str)
|
|
93
|
+
ast_commands = ast.respond_to?(:commands) ? ast.commands.to_a : []
|
|
94
|
+
write_targets = ast_commands.flat_map do |c|
|
|
95
|
+
targets = Array(c.respond_to?(:write_paths) ? c.write_paths : nil)
|
|
96
|
+
cmd_name = c.respond_to?(:command) ? c.command.to_a.first.to_s : ""
|
|
97
|
+
targets.concat(c.positionals.to_a) if ["cp", "mv", "tee", "touch", "ln", "install"].include?(cmd_name) && c.respond_to?(:positionals)
|
|
98
|
+
targets
|
|
99
|
+
end.compact.reject(&:empty?)
|
|
100
|
+
|
|
101
|
+
write_targets.concat(ast.write_paths.to_a) if ast.respond_to?(:write_paths)
|
|
102
|
+
|
|
103
|
+
write_targets.uniq.each do |wp|
|
|
104
|
+
next unless critical_path?(wp)
|
|
105
|
+
|
|
106
|
+
@audit.log_audit("blocked_command", command: cmd_str, reason: "critical_path_write", path: wp)
|
|
107
|
+
raise BlockedWriteError.new(
|
|
108
|
+
"Command write to critical path blocked: #{wp} in `#{cmd_str}`",
|
|
109
|
+
kind: :critical_path_write,
|
|
110
|
+
blocked_command: cmd_str,
|
|
111
|
+
path: wp
|
|
112
|
+
)
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "errors"
|
|
4
|
+
|
|
5
|
+
module Rubik
|
|
6
|
+
module Security
|
|
7
|
+
# The synthesis deny surface, split out of Guard (#403): the capability
|
|
8
|
+
# table (the ONE declaration point), its derived projection constants,
|
|
9
|
+
# the regex layer, the RubyVM AST walk, and the spawn-boundary entry
|
|
10
|
+
# (run_synthesis_gate). Receives the shared AuditLog instance — every
|
|
11
|
+
# refusal is audit-logged there.
|
|
12
|
+
class SynthesisGate
|
|
13
|
+
# Synth-unsafe patterns. These catch RCE-y escapes a synthesis step might
|
|
14
|
+
# smuggle past the classifier (tuned for shell commands, not Ruby):
|
|
15
|
+
# `method(:x).call`, variable send/public_send/__send__, eval/binding.eval,
|
|
16
|
+
# and backticks/%x. The interpreter/eval lines precede the backtick and %x
|
|
17
|
+
# patterns so strings merely *mentioning* backticks aren't mislabelled.
|
|
18
|
+
# Symbol-level denials live in SYNTHESIS_CAPABILITIES below; the two
|
|
19
|
+
# layers are equivalence-locked by test_synthesis_capability_coverage.rb.
|
|
20
|
+
DANGEROUS_SYNTHESIS_PATTERNS = [
|
|
21
|
+
/\b(?:send|public_send|__send__)\s*[( \s]/,
|
|
22
|
+
/\.(?:send|public_send|__send__)\b/,
|
|
23
|
+
/\b(?:method|public_method|singleton_method)\s*(?:\(\s*[:"']|\s+[:"'])/i,
|
|
24
|
+
/\beval\s*[( \s]/,
|
|
25
|
+
/\bbinding\s*\.\s*eval\b/i,
|
|
26
|
+
/\b(?:instance_eval|class_eval|module_eval)\b/,
|
|
27
|
+
/\b(?:instance_variable_get|instance_variable_set)\b/,
|
|
28
|
+
/\b(?:const_set|const_get|remove_const)\b/,
|
|
29
|
+
/(?:^|[;{\n\r])\s*(?:system|exec|spawn)\b|\b(?:system|exec|spawn)\s*\(/,
|
|
30
|
+
/(?:^|[;{\n\r])\s*fork\b|\bfork\s*(?:do|\{|\()/,
|
|
31
|
+
/\bsyscall\s*\(/,
|
|
32
|
+
/\bexit!/,
|
|
33
|
+
/\bIO\s*\.\s*popen\b/,
|
|
34
|
+
/\bOpen3\s*\.\s*(?:capture2|capture2e|capture3|popen2|popen2e|popen3|pipeline)\b/,
|
|
35
|
+
/\bProcess\s*\.\s*(?:kill|fork|daemon|spawn|exec)\b/,
|
|
36
|
+
/\bKernel\s*\.\s*(?:exec|system|fork|spawn|syscall|exit!)\b/,
|
|
37
|
+
# Unmanaged background threads survive context disposal — the exact
|
|
38
|
+
# "Leaked Side Effects" anti-pattern (AGENTS.md §5).
|
|
39
|
+
/\bThread\s*\.\s*(?:new|fork|start)\b/,
|
|
40
|
+
/\bPathname\s*\(/,
|
|
41
|
+
/`[^`]*`/,
|
|
42
|
+
Regexp.new('%x\s*[({\[<|/]', Regexp::IGNORECASE)
|
|
43
|
+
].freeze
|
|
44
|
+
|
|
45
|
+
# ── Synthesis capability table (Candidate 07) ───────────────────────
|
|
46
|
+
# The ONE declaration point for everything the synthesis AST walk
|
|
47
|
+
# denies (docs/explanation/architecture.md §2 "single declaration point" doctrine; §8.3
|
|
48
|
+
# "the seam is enforced where it is declared"). Each row is a capability
|
|
49
|
+
# a synthesized tool must never exercise; layer fields say WHERE the
|
|
50
|
+
# denial bites:
|
|
51
|
+
# nodes — RubyVM node types refused on sight
|
|
52
|
+
# calls — call names refused on ANY call shape (:fcall/:vcall
|
|
53
|
+
# AND :call/:qcall/:attrasgn)
|
|
54
|
+
# bare_calls — names refused on :fcall/:vcall only; receiver forms
|
|
55
|
+
# stay legal (ctx.require is the coeffect accessor)
|
|
56
|
+
# consts — constants refused as call RECEIVERS only
|
|
57
|
+
# any_const — constants refused on ANY :const/:colon3 reference,
|
|
58
|
+
# anti `X = TCPSocket; X.new` aliasing
|
|
59
|
+
# The FORBIDDEN_SYNTHESIS_* constants below are derived projections of
|
|
60
|
+
# this table, kept under their historical names for the walk's call
|
|
61
|
+
# sites; test_synthesis_capability_coverage.rb locks table <-> constant
|
|
62
|
+
# <-> behavior equivalence so the derivation cannot drift. Note: the
|
|
63
|
+
# :block node (grouped expressions) intentionally has NO row —
|
|
64
|
+
# `(a || b).to_i`-style expressions must stay allowed.
|
|
65
|
+
SynthesisCapability = Struct.new(:name, :reason, :nodes, :calls,
|
|
66
|
+
:bare_calls, :consts, :any_const,
|
|
67
|
+
keyword_init: true)
|
|
68
|
+
|
|
69
|
+
SYNTHESIS_CAPABILITIES = [
|
|
70
|
+
SynthesisCapability.new(
|
|
71
|
+
name: :shell_execution,
|
|
72
|
+
reason: "backtick/%x shell escapes execute outside every Ruby-level guard",
|
|
73
|
+
nodes: [:xstr, :dxstr]
|
|
74
|
+
),
|
|
75
|
+
SynthesisCapability.new(
|
|
76
|
+
name: :code_definition,
|
|
77
|
+
reason: "method/class definitions mutate the eval target; synthesized payloads are single lambdas",
|
|
78
|
+
nodes: [:defn, :defs, :class, :sclass]
|
|
79
|
+
),
|
|
80
|
+
SynthesisCapability.new(
|
|
81
|
+
name: :method_table_mutation,
|
|
82
|
+
reason: "alias/undef rewrite the method table behind the guard",
|
|
83
|
+
nodes: [:alias, :undef]
|
|
84
|
+
),
|
|
85
|
+
SynthesisCapability.new(
|
|
86
|
+
name: :global_state,
|
|
87
|
+
reason: "global variables are process-wide ambient state the host context cannot revert",
|
|
88
|
+
nodes: [:gvar, :gasgn]
|
|
89
|
+
),
|
|
90
|
+
SynthesisCapability.new(
|
|
91
|
+
name: :dynamic_execution,
|
|
92
|
+
reason: "string eval and Binding turn data into code outside the AST walk",
|
|
93
|
+
calls: [:eval, :instance_eval, :class_eval, :module_eval, :binding,
|
|
94
|
+
:instance_exec, :class_exec, :module_exec, :define_method, :define_singleton_method]
|
|
95
|
+
),
|
|
96
|
+
SynthesisCapability.new(
|
|
97
|
+
name: :process_control,
|
|
98
|
+
reason: "subprocess spawn and process lifecycle control escape the sandbox and audit trail; " \
|
|
99
|
+
"trap/at_exit install process-wide hooks the host cannot revert — Signal.trap can " \
|
|
100
|
+
"swallow the host REPL's Interrupt/exit handling; exit/exit! can kill the host " \
|
|
101
|
+
"when preflight runs in-process",
|
|
102
|
+
calls: [:system, :exec, :spawn, :fork, :syscall, :exit, :exit!, :trap, :at_exit,
|
|
103
|
+
:popen, :capture2, :capture2e, :capture3, :popen2, :popen2e,
|
|
104
|
+
:popen3, :pipeline, :kill, :daemon],
|
|
105
|
+
consts: [:Process, :Thread, :Signal, :Open3]
|
|
106
|
+
),
|
|
107
|
+
SynthesisCapability.new(
|
|
108
|
+
name: :reflection,
|
|
109
|
+
reason: "send/method-object families bypass static call-name checks (the method(:x).call escape)",
|
|
110
|
+
calls: [:send, :public_send, :__send__,
|
|
111
|
+
:method, :public_method, :singleton_method,
|
|
112
|
+
:instance_method, :public_instance_method, :bind, :bind_call]
|
|
113
|
+
),
|
|
114
|
+
SynthesisCapability.new(
|
|
115
|
+
name: :state_introspection,
|
|
116
|
+
reason: "object internals and object-graph walks fish for live objects and sidestep the AST deny surface (the method(:x).call family at runtime)",
|
|
117
|
+
calls: [:instance_variable_get, :instance_variable_set],
|
|
118
|
+
consts: [:ObjectSpace]
|
|
119
|
+
),
|
|
120
|
+
SynthesisCapability.new(
|
|
121
|
+
name: :namespace_mutation,
|
|
122
|
+
reason: "const_get/const_set/remove_const read and rewrite constant namespaces at runtime",
|
|
123
|
+
calls: [:const_get, :const_set, :remove_const]
|
|
124
|
+
),
|
|
125
|
+
SynthesisCapability.new(
|
|
126
|
+
name: :filesystem,
|
|
127
|
+
reason: "raw file/IO handles bypass the OutputView and every guard write check",
|
|
128
|
+
calls: [:open, :read, :write, :binread, :binwrite, :unlink, :rm_rf, :rmtree,
|
|
129
|
+
:sysopen, :syswrite, :sysread, :readlines, :foreach, :Pathname],
|
|
130
|
+
consts: [:IO, :File, :Dir, :FileUtils, :STDIN, :STDOUT, :STDERR, :ARGF, :Pathname]
|
|
131
|
+
),
|
|
132
|
+
SynthesisCapability.new(
|
|
133
|
+
name: :code_loading,
|
|
134
|
+
reason: "require/load execute files outside every guard. :require/:require_relative must NOT live in calls: " \
|
|
135
|
+
"ctx.require('database') is the canonical coeffect accessor (receiver call) and every synthesized " \
|
|
136
|
+
"tool depends on it — the bare forms are the escape. Kernel.require/Kernel.load are already caught " \
|
|
137
|
+
"via :Kernel in runtime_root; obj.load would load arbitrary code or deserialize untrusted state",
|
|
138
|
+
calls: [:load, :autoload],
|
|
139
|
+
bare_calls: [:require, :require_relative, :load, :autoload]
|
|
140
|
+
),
|
|
141
|
+
SynthesisCapability.new(
|
|
142
|
+
name: :runtime_root,
|
|
143
|
+
reason: "core runtime singletons (Kernel/RubyVM/Class/Module/...) re-open every denied capability",
|
|
144
|
+
consts: [:Kernel, :RubyVM, :Object, :Class, :Module, :BasicObject, :Rubik, :Fiber, :TracePoint, :GC]
|
|
145
|
+
),
|
|
146
|
+
SynthesisCapability.new(
|
|
147
|
+
name: :ambient_state,
|
|
148
|
+
reason: "ENV reads expose provider keys / ambient config and mutate process-wide state the host cannot revert",
|
|
149
|
+
consts: [:ENV]
|
|
150
|
+
),
|
|
151
|
+
SynthesisCapability.new(
|
|
152
|
+
name: :network_serialization,
|
|
153
|
+
reason: "sockets and Marshal have no legitimate synthesized-tool use; receiver-only checks leave " \
|
|
154
|
+
"`X = TCPSocket; X.new` (:casgn aliasing) wide open, so their mere appearance is a refusal",
|
|
155
|
+
any_const: [:Socket, :IPSocket, :TCPSocket, :TCPServer, :UDPSocket,
|
|
156
|
+
:UNIXSocket, :UNIXServer, :Net, :Marshal]
|
|
157
|
+
)
|
|
158
|
+
].map do |cap|
|
|
159
|
+
# Deep-freeze: the ONE declaration point for a security gate must be
|
|
160
|
+
# immutable by construction, not by convention — Struct#freeze alone
|
|
161
|
+
# leaves the member arrays mutable.
|
|
162
|
+
cap.members.each { |member| cap[member].freeze if cap[member].is_a?(Array) }
|
|
163
|
+
cap.freeze
|
|
164
|
+
end.freeze
|
|
165
|
+
|
|
166
|
+
# Fold one capability layer into its denial list. Load-time, pure;
|
|
167
|
+
# test_synthesis_capability_coverage.rb pins this helper as the ONLY
|
|
168
|
+
# fold from table to denial constants.
|
|
169
|
+
def self.synthesis_capability_layer(field)
|
|
170
|
+
SYNTHESIS_CAPABILITIES.flat_map { |cap| cap[field].to_a }.freeze
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
FORBIDDEN_SYNTHESIS_NODES = synthesis_capability_layer(:nodes)
|
|
174
|
+
FORBIDDEN_SYNTHESIS_CALLS = synthesis_capability_layer(:calls)
|
|
175
|
+
FORBIDDEN_SYNTHESIS_BARE_CALLS = synthesis_capability_layer(:bare_calls)
|
|
176
|
+
FORBIDDEN_SYNTHESIS_CONSTS = synthesis_capability_layer(:consts)
|
|
177
|
+
FORBIDDEN_SYNTHESIS_NETWORK_CONSTS = synthesis_capability_layer(:any_const)
|
|
178
|
+
|
|
179
|
+
# The ONE synthesis gate: every site about to eval synthesized code
|
|
180
|
+
# comes through here (synthesis builtins AND PreflightRunner's
|
|
181
|
+
# spawn-boundary re-check). A guard lacking verify_synthesis_safety!
|
|
182
|
+
# is downgraded to the shell-only gate — the duck-typed fallback is a
|
|
183
|
+
# test-double affordance, but skipping the entire AST deny surface is
|
|
184
|
+
# security-observable, so the downgrade is loud. $stderr.puts, not
|
|
185
|
+
# Kernel#warn: warn is silenced when $VERBOSE is nil (bundler/rake
|
|
186
|
+
# -W0) — same ruling as Log.emit. A failing emit is fail-closed AND
|
|
187
|
+
# honest: refusal with the real cause, never a misattributed
|
|
188
|
+
# "dangerous operations".
|
|
189
|
+
def self.run_synthesis_gate(guard, code)
|
|
190
|
+
if guard.respond_to?(:verify_synthesis_safety!)
|
|
191
|
+
guard.verify_synthesis_safety!(code)
|
|
192
|
+
else
|
|
193
|
+
begin
|
|
194
|
+
$stderr.puts "[Synthesize] guard lacks verify_synthesis_safety!; downgraded to shell-only gate" # rubocop:disable Style/StderrPuts
|
|
195
|
+
rescue StandardError => e
|
|
196
|
+
raise BlockedWriteError,
|
|
197
|
+
"guard downgrade warn could not be written to stderr (#{e.class}: #{e.message}); refusing to eval"
|
|
198
|
+
end
|
|
199
|
+
guard.verify_command_safety!(code)
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def initialize(audit:)
|
|
204
|
+
@audit = audit
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Reject synthesized Ruby before it is ever eval'd.
|
|
208
|
+
#
|
|
209
|
+
# Defense-in-depth architecture:
|
|
210
|
+
# 1. Fast regex pattern matching (DANGEROUS_SYNTHESIS_PATTERNS) intercepts known keywords.
|
|
211
|
+
# 2. RubyVM::AbstractSyntaxTree semantic walk checks normalized node types, method calls,
|
|
212
|
+
# and receiver constants against explicit deny rules to block AST-level evasion vectors.
|
|
213
|
+
#
|
|
214
|
+
# Raises Rubik::Security::BlockedWriteError with an actionable message.
|
|
215
|
+
def verify_synthesis_safety!(code)
|
|
216
|
+
code_str = code.to_s
|
|
217
|
+
|
|
218
|
+
matched = DANGEROUS_SYNTHESIS_PATTERNS.find { |p| code_str.match?(p) }
|
|
219
|
+
if matched
|
|
220
|
+
@audit.log_audit("blocked_synthesis", code: code_str, reason: "dangerous_synthesis_pattern")
|
|
221
|
+
raise BlockedWriteError, "Dangerous synthesis code blocked by security guard: #{matched.inspect}"
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
synthesize_safety_walk(code_str)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def synthesize_safety_walk(code_str)
|
|
228
|
+
ast = begin
|
|
229
|
+
RubyVM::AbstractSyntaxTree.parse(code_str)
|
|
230
|
+
rescue ArgumentError, SyntaxError
|
|
231
|
+
# Code with syntax errors cannot execute; let synthesis eval safely report
|
|
232
|
+
# a SyntaxError to the user rather than misclassifying a syntax typo as a security breach.
|
|
233
|
+
nil
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
return true if ast.nil?
|
|
237
|
+
|
|
238
|
+
synthesize_safety_check_node(ast)
|
|
239
|
+
|
|
240
|
+
true
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Recursively walk the AST, refusing if any disallowed construct is encountered.
|
|
244
|
+
def synthesize_safety_check_node(node)
|
|
245
|
+
return true unless node.respond_to?(:type)
|
|
246
|
+
|
|
247
|
+
type = node.type.to_s.downcase.to_sym
|
|
248
|
+
reject_unsafe(node, "node type :#{type}") if FORBIDDEN_SYNTHESIS_NODES.include?(type)
|
|
249
|
+
|
|
250
|
+
case type
|
|
251
|
+
when :const, :colon3
|
|
252
|
+
# :const covers Net::HTTP / Marshal.dump receivers, `X = Socket`
|
|
253
|
+
# RHS values, and local aliasing `k = Kernel`; :colon3 covers top-level scoped `::TCPSocket`.
|
|
254
|
+
name = node.children[0]
|
|
255
|
+
forbidden_const = FORBIDDEN_SYNTHESIS_NETWORK_CONSTS.include?(name) || FORBIDDEN_SYNTHESIS_CONSTS.include?(name)
|
|
256
|
+
reject_unsafe(node, "reference to forbidden constant :#{name}") if forbidden_const
|
|
257
|
+
when :colon2
|
|
258
|
+
# Scoped constant reference like Rubik::Security::Guard
|
|
259
|
+
name = node.children[1]
|
|
260
|
+
forbidden_const = FORBIDDEN_SYNTHESIS_NETWORK_CONSTS.include?(name) || FORBIDDEN_SYNTHESIS_CONSTS.include?(name)
|
|
261
|
+
reject_unsafe(node, "reference to forbidden constant :#{name}") if forbidden_const
|
|
262
|
+
when :fcall, :vcall
|
|
263
|
+
name = node.children[0]
|
|
264
|
+
reject_unsafe(node, "call to :#{name}") if FORBIDDEN_SYNTHESIS_CALLS.include?(name)
|
|
265
|
+
reject_unsafe(node, "bare call to :#{name}") if FORBIDDEN_SYNTHESIS_BARE_CALLS.include?(name)
|
|
266
|
+
when :call, :qcall, :attrasgn
|
|
267
|
+
recv = node.children[0]
|
|
268
|
+
name = node.children[1]
|
|
269
|
+
|
|
270
|
+
reject_unsafe(node, "call to :#{name}") if FORBIDDEN_SYNTHESIS_CALLS.include?(name)
|
|
271
|
+
|
|
272
|
+
# Receiver const check must accept const node shapes: :const, :colon3, and :colon2
|
|
273
|
+
if recv.respond_to?(:type)
|
|
274
|
+
recv_type = recv.type.to_s.downcase.to_sym
|
|
275
|
+
if [:const, :colon3].include?(recv_type)
|
|
276
|
+
const_name = recv.children[0]
|
|
277
|
+
reject_unsafe(node, "access to #{const_name}.#{name}") if FORBIDDEN_SYNTHESIS_CONSTS.include?(const_name)
|
|
278
|
+
elsif recv_type == :colon2
|
|
279
|
+
const_name = recv.children[1]
|
|
280
|
+
reject_unsafe(node, "access to #{const_name}.#{name}") if FORBIDDEN_SYNTHESIS_CONSTS.include?(const_name)
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
node.children.compact.each do |c|
|
|
286
|
+
synthesize_safety_check_node(c) if c.respond_to?(:type)
|
|
287
|
+
end
|
|
288
|
+
true
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def reject_unsafe(node, detail = nil)
|
|
292
|
+
kind = node.type.inspect
|
|
293
|
+
reason = detail ? "unsafe_#{detail.to_s.gsub(/\s+/, '_')}" : "unsafe_node_#{kind}"
|
|
294
|
+
@audit.log_audit("blocked_synthesis", code: node.children[0].to_s, reason: reason)
|
|
295
|
+
raise BlockedWriteError,
|
|
296
|
+
"Synthesized code was refused by the security guard: #{detail || "a #{kind} node"} is not permitted in synthesized tools."
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
end
|
|
300
|
+
end
|
data/lib/rubik/security.rb
CHANGED
|
@@ -2,6 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "security/errors"
|
|
4
4
|
require_relative "security/redactor"
|
|
5
|
+
require_relative "security/audit_log"
|
|
6
|
+
require_relative "security/allowlist"
|
|
7
|
+
require_relative "security/path_policy"
|
|
8
|
+
require_relative "security/synthesis_gate"
|
|
9
|
+
require_relative "security/command_gate"
|
|
5
10
|
require_relative "security/guard"
|
|
6
11
|
require_relative "security/classifier"
|
|
7
12
|
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rubik
|
|
4
|
+
module Storage
|
|
5
|
+
# BM25-lite relevance ranking (vakanuvis-v2 §4-V8, issue #408): the
|
|
6
|
+
# single home of the scored-search math. Store keeps tokenization, the
|
|
7
|
+
# shared @records_cache and corpus assembly; Ranking owns the candidate
|
|
8
|
+
# gate, corpus statistics (total/avgdl/df), Okapi scoring, the field
|
|
9
|
+
# tier bonus and the final ordering. `rank` is pure over its inputs —
|
|
10
|
+
# same corpus + same tokens + same require_all ⇒ same order, every
|
|
11
|
+
# replay (replay/pin contract below).
|
|
12
|
+
module Ranking
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
# BM25-lite (vakanüvis-v2 §4-V8): klasik Okapi değerleri — k1 terim
|
|
16
|
+
# doygunluğu, b uzunluk normalizasyonu ağırlığı.
|
|
17
|
+
BM25_K1 = 1.2
|
|
18
|
+
BM25_B = 0.75
|
|
19
|
+
# Alan kademeli bonus (issue #99 P3): BM25 birincil kalır; başlıkta
|
|
20
|
+
# geçen sorgu terimi etikette geçene, o da yalnız gövdede geçene
|
|
21
|
+
# üstünlük kurar. Terim başına birimdir ve tipik idf katkısının
|
|
22
|
+
# altındadır — maddesel BM25 farklarını ezmez, sadece eşit-yakın
|
|
23
|
+
# skorları alan bilgisiyle kırar.
|
|
24
|
+
TITLE_TIER_BONUS = 1.0
|
|
25
|
+
TAGS_TIER_BONUS = 0.5
|
|
26
|
+
|
|
27
|
+
# One public entry. Store#search_ranked supplies the corpus entries —
|
|
28
|
+
# each `{records:, haystacks:, folded_fields:, has:}` where `has` is
|
|
29
|
+
# the per-token include? matrix produced by the single AND/OR pass —
|
|
30
|
+
# so df comes from that pass alone and tf is scanned only on matching
|
|
31
|
+
# records. Ordering: score desc → updated_at desc → [type, row].
|
|
32
|
+
def rank(tokens:, corpus:, require_all:)
|
|
33
|
+
return [] if corpus.empty?
|
|
34
|
+
|
|
35
|
+
total = corpus.sum { |c| c[:records].size }
|
|
36
|
+
avgdl = corpus.sum { |c| c[:haystacks].sum(&:bytesize) }.fdiv(total)
|
|
37
|
+
dfs = tokens.each_index.map { |ti| corpus.sum { |c| c[:has][ti].count(true) } }
|
|
38
|
+
|
|
39
|
+
gate = ranked_gate(require_all)
|
|
40
|
+
scored = corpus.flat_map do |c|
|
|
41
|
+
c[:records].each_index.select { |i| gate.call(c[:has], i) }.map do |i|
|
|
42
|
+
score = tokens.each_with_index.sum do |token, ti|
|
|
43
|
+
df = dfs[ti]
|
|
44
|
+
next 0.0 if df.zero?
|
|
45
|
+
|
|
46
|
+
# PERF: count_occurrences using String#index avoids Array allocation of String#scan
|
|
47
|
+
tf = count_occurrences(c[:haystacks][i], token)
|
|
48
|
+
next 0.0 if tf.zero?
|
|
49
|
+
|
|
50
|
+
idf = Math.log(((total - df + 0.5) / (df + 0.5)) + 1)
|
|
51
|
+
dl = c[:haystacks][i].bytesize
|
|
52
|
+
idf * (tf * (BM25_K1 + 1)) / (tf + (BM25_K1 * (1 - BM25_B + (BM25_B * dl / avgdl))))
|
|
53
|
+
end
|
|
54
|
+
# PERF: pass cached folded title and tags to avoid String folding & array join allocations per score
|
|
55
|
+
f_title, f_tags = c[:folded_fields][i]
|
|
56
|
+
[score + field_tier_bonus(tokens, f_title, f_tags), c[:records][i]]
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
ordered = scored.sort do |(score_a, rec_a), (score_b, rec_b)|
|
|
60
|
+
(score_b <=> score_a).nonzero? ||
|
|
61
|
+
(rec_b["updated_at"].to_s <=> rec_a["updated_at"].to_s).nonzero? ||
|
|
62
|
+
(rec_a["type"].to_s <=> rec_b["type"].to_s).nonzero? ||
|
|
63
|
+
(rec_a["row"].to_i <=> rec_b["row"].to_i)
|
|
64
|
+
end
|
|
65
|
+
ordered.map { |_score, record| record }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Aday eşiği: require_all (AND — default, recall) ya da OR
|
|
69
|
+
# gevşetmesi (fallback — issue #99 P2).
|
|
70
|
+
def ranked_gate(require_all)
|
|
71
|
+
if require_all
|
|
72
|
+
->(has, i) { has.all? { |row| row[i] } }
|
|
73
|
+
else
|
|
74
|
+
->(has, i) { has.any? { |row| row[i] } }
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# PERF: Zero-allocation string occurrence counting via String#index loop.
|
|
79
|
+
# Replaces String#scan(token).size to avoid array allocation per matched term.
|
|
80
|
+
def count_occurrences(haystack, token)
|
|
81
|
+
return 0 if token.empty?
|
|
82
|
+
|
|
83
|
+
count = 0
|
|
84
|
+
offset = 0
|
|
85
|
+
token_len = token.length
|
|
86
|
+
while (idx = haystack.index(token, offset))
|
|
87
|
+
count += 1
|
|
88
|
+
offset = idx + token_len
|
|
89
|
+
end
|
|
90
|
+
count
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Terim başına alan izi: katlanmış başlıkta geçen terim +1.0,
|
|
94
|
+
# yalnız katlanmış etiketlerde geçen +0.5, gövde izi 0 — bkz.
|
|
95
|
+
# TITLE_TIER_BONUS/TAGS_TIER_BONUS. Yalnız eşik geçen kayıtlar için
|
|
96
|
+
# hesaplanır (ek tarama yok); başlık/etiket önceden katlanmış
|
|
97
|
+
# dizgeler olarak corpus'tan gelir (Store memoize'ı).
|
|
98
|
+
def field_tier_bonus(tokens, title, tags)
|
|
99
|
+
sum = 0.0
|
|
100
|
+
tokens.each do |token|
|
|
101
|
+
if title.include?(token)
|
|
102
|
+
sum += TITLE_TIER_BONUS
|
|
103
|
+
elsif tags.include?(token)
|
|
104
|
+
sum += TAGS_TIER_BONUS
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
sum
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
private_class_method :ranked_gate, :count_occurrences, :field_tier_bonus
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
data/lib/rubik/storage/store.rb
CHANGED
|
@@ -9,6 +9,7 @@ require_relative "../platform"
|
|
|
9
9
|
require_relative "../uris"
|
|
10
10
|
require_relative "layout"
|
|
11
11
|
require_relative "folding"
|
|
12
|
+
require_relative "ranking"
|
|
12
13
|
|
|
13
14
|
module Rubik
|
|
14
15
|
module Storage
|
|
@@ -32,17 +33,6 @@ module Rubik
|
|
|
32
33
|
ID_PATTERN = /\A([a-z]+)\.(\d+)\.([0-9a-f]{16})\z/
|
|
33
34
|
HASH_PREFIX_MIN = 8
|
|
34
35
|
REF_PATTERN = /\A[a-zA-Z0-9._-]+\z/
|
|
35
|
-
# BM25-lite (vakanüvis-v2 §4-V8): klasik Okapi değerleri — k1 terim
|
|
36
|
-
# doygunluğu, b uzunluk normalizasyonu ağırlığı.
|
|
37
|
-
BM25_K1 = 1.2
|
|
38
|
-
BM25_B = 0.75
|
|
39
|
-
# Alan kademeli bonus (issue #99 P3): BM25 birincil kalır; başlıkta
|
|
40
|
-
# geçen sorgu terimi etikette geçene, o da yalnız gövdede geçene
|
|
41
|
-
# üstünlük kurar. Terim başına birimdir ve tipik idf katkısının
|
|
42
|
-
# altındadır — maddesel BM25 farklarını ezmez, sadece eşit-yakın
|
|
43
|
-
# skorları alan bilgisiyle kırar.
|
|
44
|
-
TITLE_TIER_BONUS = 1.0
|
|
45
|
-
TAGS_TIER_BONUS = 0.5
|
|
46
36
|
|
|
47
37
|
Summary = Struct.new(:id, :row, :title, :status, :updated_at, :tags, keyword_init: true)
|
|
48
38
|
|
|
@@ -184,88 +174,8 @@ module Rubik
|
|
|
184
174
|
end
|
|
185
175
|
return [] if corpus.empty?
|
|
186
176
|
|
|
187
|
-
|
|
188
|
-
avgdl = corpus.sum { |c| c[:haystacks].sum(&:bytesize) }.fdiv(total)
|
|
189
|
-
dfs = tokens.each_index.map { |ti| corpus.sum { |c| c[:has][ti].count(true) } }
|
|
190
|
-
|
|
191
|
-
gate = ranked_gate(require_all)
|
|
192
|
-
scored = corpus.flat_map do |c|
|
|
193
|
-
c[:records].each_index.select { |i| gate.call(c[:has], i) }.map do |i|
|
|
194
|
-
score = tokens.each_with_index.sum do |token, ti|
|
|
195
|
-
df = dfs[ti]
|
|
196
|
-
next 0.0 if df.zero?
|
|
197
|
-
|
|
198
|
-
# PERF: count_occurrences using String#index avoids Array allocation of String#scan
|
|
199
|
-
tf = count_occurrences(c[:haystacks][i], token)
|
|
200
|
-
next 0.0 if tf.zero?
|
|
201
|
-
|
|
202
|
-
idf = Math.log(((total - df + 0.5) / (df + 0.5)) + 1)
|
|
203
|
-
dl = c[:haystacks][i].bytesize
|
|
204
|
-
idf * (tf * (BM25_K1 + 1)) / (tf + (BM25_K1 * (1 - BM25_B + (BM25_B * dl / avgdl))))
|
|
205
|
-
end
|
|
206
|
-
# PERF: pass cached folded title and tags to avoid String folding & array join allocations per score
|
|
207
|
-
f_title, f_tags = c[:folded_fields][i]
|
|
208
|
-
[score + field_tier_bonus(tokens, f_title, f_tags), c[:records][i]]
|
|
209
|
-
end
|
|
210
|
-
end
|
|
211
|
-
ordered = scored.sort do |(score_a, rec_a), (score_b, rec_b)|
|
|
212
|
-
(score_b <=> score_a).nonzero? ||
|
|
213
|
-
(rec_b["updated_at"].to_s <=> rec_a["updated_at"].to_s).nonzero? ||
|
|
214
|
-
(rec_a["type"].to_s <=> rec_b["type"].to_s).nonzero? ||
|
|
215
|
-
(rec_a["row"].to_i <=> rec_b["row"].to_i)
|
|
216
|
-
end
|
|
217
|
-
ordered.map { |_score, record| record }
|
|
218
|
-
end
|
|
219
|
-
end
|
|
220
|
-
|
|
221
|
-
# Aday eşiği: require_all (AND — default, recall) ya da OR
|
|
222
|
-
# gevşetmesi (fallback — issue #99 P2).
|
|
223
|
-
def ranked_gate(require_all)
|
|
224
|
-
if require_all
|
|
225
|
-
->(has, i) { has.all? { |row| row[i] } }
|
|
226
|
-
else
|
|
227
|
-
->(has, i) { has.any? { |row| row[i] } }
|
|
228
|
-
end
|
|
229
|
-
end
|
|
230
|
-
|
|
231
|
-
# PERF: Zero-allocation string occurrence counting via String#index loop.
|
|
232
|
-
# Replaces String#scan(token).size to avoid array allocation per matched term.
|
|
233
|
-
def count_occurrences(haystack, token)
|
|
234
|
-
return 0 if token.empty?
|
|
235
|
-
|
|
236
|
-
count = 0
|
|
237
|
-
offset = 0
|
|
238
|
-
token_len = token.length
|
|
239
|
-
while (idx = haystack.index(token, offset))
|
|
240
|
-
count += 1
|
|
241
|
-
offset = idx + token_len
|
|
242
|
-
end
|
|
243
|
-
count
|
|
244
|
-
end
|
|
245
|
-
|
|
246
|
-
# Terim başına alan izi: katlanmış başlıkta geçen terim +1.0,
|
|
247
|
-
# yalnız katlanmış etiketlerde geçen +0.5, gövde izi 0 — bkz.
|
|
248
|
-
# TITLE_TIER_BONUS/TAGS_TIER_BONUS. Yalnız AND eşiğini geçen
|
|
249
|
-
# kayıtlar için hesaplanır (ek tarama yok).
|
|
250
|
-
# PERF: accepts pre-folded title & tags strings or fallback record for backward compatibility.
|
|
251
|
-
def field_tier_bonus(tokens, title_or_record, tags = nil)
|
|
252
|
-
if tags.nil? && title_or_record.is_a?(Hash)
|
|
253
|
-
title = Folding.fold(title_or_record["title"])
|
|
254
|
-
tags = Folding.fold(Array(title_or_record["tags"]).join("\n"))
|
|
255
|
-
else
|
|
256
|
-
title = title_or_record.to_s
|
|
257
|
-
tags = tags.to_s
|
|
258
|
-
end
|
|
259
|
-
|
|
260
|
-
sum = 0.0
|
|
261
|
-
tokens.each do |token|
|
|
262
|
-
if title.include?(token)
|
|
263
|
-
sum += TITLE_TIER_BONUS
|
|
264
|
-
elsif tags.include?(token)
|
|
265
|
-
sum += TAGS_TIER_BONUS
|
|
266
|
-
end
|
|
177
|
+
Ranking.rank(tokens: tokens, corpus: corpus, require_all: require_all)
|
|
267
178
|
end
|
|
268
|
-
sum
|
|
269
179
|
end
|
|
270
180
|
|
|
271
181
|
def set_ref(name, id)
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
4
|
require_relative "../security/redactor"
|
|
5
|
+
require_relative "../tools/truncator"
|
|
5
6
|
require_relative "../uris"
|
|
6
7
|
|
|
7
8
|
module Rubik
|
|
@@ -117,9 +118,7 @@ module Rubik
|
|
|
117
118
|
|
|
118
119
|
def clip(text)
|
|
119
120
|
text = text.to_s.scrub
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
"#{text.slice(0, PART_LIMIT)}\n... [truncated #{text.length - PART_LIMIT} chars]"
|
|
121
|
+
Rubik::Tools::Truncator.clip(text, PART_LIMIT, separator: "\n")
|
|
123
122
|
end
|
|
124
123
|
end
|
|
125
124
|
end
|