woods 1.6.1 → 2.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (274) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2035 -0
  3. data/CONTRIBUTING.md +253 -87
  4. data/README.md +161 -513
  5. data/SECURITY.md +92 -0
  6. data/assets/woods-wordmark-white-with-bg.png +0 -0
  7. data/docs/AGENT_GUIDE.md +204 -0
  8. data/docs/AGENT_SETUP.md +205 -0
  9. data/docs/BACKEND_MATRIX.md +470 -0
  10. data/docs/CONFIGURATION_REFERENCE.md +655 -0
  11. data/docs/CONSOLE_MCP_SETUP.md +829 -0
  12. data/docs/DOCKER_SETUP.md +454 -0
  13. data/docs/EMBEDDING_MODELS.md +136 -0
  14. data/docs/EVALUATION.md +91 -0
  15. data/docs/EXTRACTOR_REFERENCE.md +765 -0
  16. data/docs/FAQ.md +544 -0
  17. data/docs/GETTING_STARTED.md +183 -0
  18. data/docs/INCREMENTAL_EXTRACTION.md +455 -0
  19. data/docs/INTERNALS.md +418 -0
  20. data/docs/MCP_HTTP_TRANSPORT.md +144 -0
  21. data/docs/MCP_SERVERS.md +231 -0
  22. data/docs/MCP_TOOL_COOKBOOK.md +987 -0
  23. data/docs/MCP_WORKTREE_SETUP.md +127 -0
  24. data/docs/NOTION_INTEGRATION.md +283 -0
  25. data/docs/OBSIDIAN_INTEGRATION.md +170 -0
  26. data/docs/PUBLISHED_INDEX.md +213 -0
  27. data/docs/README.md +94 -0
  28. data/docs/RETRIEVAL_GUIDE.md +267 -0
  29. data/docs/TOKEN_BENCHMARK.md +68 -0
  30. data/docs/TROUBLESHOOTING.md +841 -0
  31. data/docs/UNBLOCKED_INTEGRATION.md +279 -0
  32. data/docs/UPGRADING_TO_2.md +321 -0
  33. data/docs/WATCH_DAEMON.md +667 -0
  34. data/docs/WHY_WOODS.md +219 -0
  35. data/exe/woods-console +40 -4
  36. data/exe/woods-console-mcp +21 -35
  37. data/exe/woods-mcp +20 -7
  38. data/exe/woods-mcp-http +80 -11
  39. data/exe/woods-mcp-start +57 -52
  40. data/lib/generators/woods/install_generator.rb +6 -5
  41. data/lib/generators/woods/pgvector_generator.rb +6 -3
  42. data/lib/generators/woods/templates/add_pgvector_to_woods.rb.erb +29 -9
  43. data/lib/generators/woods/templates/create_woods_tables.rb.erb +5 -1
  44. data/lib/generators/woods/templates/woods.rb.tt +49 -28
  45. data/lib/tasks/woods.rake +622 -168
  46. data/lib/tasks/woods_checks.rake +107 -0
  47. data/lib/tasks/woods_evaluation.rake +164 -80
  48. data/lib/woods/ast/call_site_extractor.rb +6 -15
  49. data/lib/woods/ast/method_extractor.rb +19 -9
  50. data/lib/woods/ast/parser.rb +54 -8
  51. data/lib/woods/atomic_file.rb +171 -2
  52. data/lib/woods/builder.rb +310 -22
  53. data/lib/woods/cache/cache_middleware.rb +7 -2
  54. data/lib/woods/cache/cache_store.rb +9 -1
  55. data/lib/woods/cache/solid_cache_store.rb +6 -4
  56. data/lib/woods/change_set.rb +88 -0
  57. data/lib/woods/checks/generation_resolution.rb +34 -0
  58. data/lib/woods/checks/moved_messages.rb +186 -0
  59. data/lib/woods/chunking/semantic_chunker.rb +160 -18
  60. data/lib/woods/console/audit_logger.rb +12 -3
  61. data/lib/woods/console/bridge_protocol.rb +3 -16
  62. data/lib/woods/console/connection_manager.rb +51 -136
  63. data/lib/woods/console/dispatch_pipeline.rb +42 -12
  64. data/lib/woods/console/embedded_executor.rb +806 -149
  65. data/lib/woods/console/eval_guard.rb +27 -20
  66. data/lib/woods/console/input_contract.rb +78 -0
  67. data/lib/woods/console/model_validator.rb +29 -1
  68. data/lib/woods/console/rack_middleware.rb +65 -42
  69. data/lib/woods/console/redactor.rb +26 -8
  70. data/lib/woods/console/safe_context.rb +58 -10
  71. data/lib/woods/console/scope_predicate_parser.rb +41 -0
  72. data/lib/woods/console/server.rb +119 -247
  73. data/lib/woods/console/sql_noise_stripper.rb +125 -16
  74. data/lib/woods/console/sql_table_scanner.rb +82 -22
  75. data/lib/woods/console/sql_validator.rb +459 -29
  76. data/lib/woods/console/table_gate.rb +2 -2
  77. data/lib/woods/console/tool_specs.rb +463 -90
  78. data/lib/woods/console/tools/tier1.rb +1 -5
  79. data/lib/woods/console/tools/tier4.rb +18 -9
  80. data/lib/woods/coordination/lock_heartbeat.rb +103 -0
  81. data/lib/woods/coordination/pipeline_lock.rb +263 -53
  82. data/lib/woods/db/migrations/007_typed_snapshot_units.rb +45 -0
  83. data/lib/woods/db/migrator.rb +3 -9
  84. data/lib/woods/db/schema_version.rb +47 -2
  85. data/lib/woods/dependency_graph.rb +898 -64
  86. data/lib/woods/embedding/fake.rb +138 -0
  87. data/lib/woods/embedding/indexer.rb +832 -40
  88. data/lib/woods/embedding/openai.rb +77 -19
  89. data/lib/woods/embedding/provider.rb +189 -11
  90. data/lib/woods/embedding/text_preparer.rb +1 -1
  91. data/lib/woods/embedding/token_counter.rb +0 -7
  92. data/lib/woods/evaluation/ablation_agent_payload.rb +38 -0
  93. data/lib/woods/evaluation/ablation_executor.rb +67 -0
  94. data/lib/woods/evaluation/ablation_provenance.rb +38 -0
  95. data/lib/woods/evaluation/ablation_report_writer.rb +43 -0
  96. data/lib/woods/evaluation/ablation_runner.rb +173 -0
  97. data/lib/woods/evaluation/ablation_summary.rb +65 -0
  98. data/lib/woods/evaluation/ablation_task.rb +66 -0
  99. data/lib/woods/evaluation/ablation_task_set.rb +77 -0
  100. data/lib/woods/evaluation/ablation_timed_executor.rb +91 -0
  101. data/lib/woods/evaluation/ablation_worktree.rb +71 -0
  102. data/lib/woods/evaluation/baseline.rb +60 -0
  103. data/lib/woods/evaluation/baseline_runner.rb +11 -3
  104. data/lib/woods/evaluation/evaluator.rb +41 -8
  105. data/lib/woods/evaluation/query_set.rb +79 -13
  106. data/lib/woods/evaluation/report_generator.rb +20 -1
  107. data/lib/woods/export/unit_facts.rb +0 -11
  108. data/lib/woods/extracted_unit.rb +22 -63
  109. data/lib/woods/extractor.rb +2783 -238
  110. data/lib/woods/extractors/action_cable_extractor.rb +9 -4
  111. data/lib/woods/extractors/ast_source_extraction.rb +20 -2
  112. data/lib/woods/extractors/caching_extractor.rb +46 -12
  113. data/lib/woods/extractors/callback_analyzer.rb +39 -9
  114. data/lib/woods/extractors/component_discovery.rb +123 -0
  115. data/lib/woods/extractors/concern_extractor.rb +17 -3
  116. data/lib/woods/extractors/controller_extractor.rb +389 -29
  117. data/lib/woods/extractors/decorator_extractor.rb +7 -14
  118. data/lib/woods/extractors/engine_extractor.rb +53 -8
  119. data/lib/woods/extractors/event_extractor.rb +55 -4
  120. data/lib/woods/extractors/factory_extractor.rb +49 -11
  121. data/lib/woods/extractors/graphql_extractor.rb +162 -66
  122. data/lib/woods/extractors/i18n_extractor.rb +6 -1
  123. data/lib/woods/extractors/job_extractor.rb +51 -21
  124. data/lib/woods/extractors/lib_extractor.rb +23 -17
  125. data/lib/woods/extractors/line_neutralizer.rb +171 -0
  126. data/lib/woods/extractors/mailer_extractor.rb +9 -1
  127. data/lib/woods/extractors/manager_extractor.rb +19 -2
  128. data/lib/woods/extractors/migration_extractor.rb +22 -11
  129. data/lib/woods/extractors/model_extractor.rb +292 -57
  130. data/lib/woods/extractors/package_extractor.rb +154 -0
  131. data/lib/woods/extractors/phlex_extractor.rb +18 -3
  132. data/lib/woods/extractors/policy_extractor.rb +6 -5
  133. data/lib/woods/extractors/poro_extractor.rb +13 -14
  134. data/lib/woods/extractors/pundit_extractor.rb +3 -3
  135. data/lib/woods/extractors/rails_source_extractor.rb +24 -7
  136. data/lib/woods/extractors/rake_task_extractor.rb +158 -30
  137. data/lib/woods/extractors/reference_patterns.rb +38 -0
  138. data/lib/woods/extractors/route_extractor.rb +58 -2
  139. data/lib/woods/extractors/scheduled_job_extractor.rb +51 -35
  140. data/lib/woods/extractors/serializer_extractor.rb +3 -4
  141. data/lib/woods/extractors/service_extractor.rb +11 -1
  142. data/lib/woods/extractors/shared_dependency_scanner.rb +24 -34
  143. data/lib/woods/extractors/shared_utility_methods.rb +36 -6
  144. data/lib/woods/extractors/source_nesting.rb +560 -0
  145. data/lib/woods/extractors/state_machine_extractor.rb +30 -18
  146. data/lib/woods/extractors/test_mapping_extractor.rb +26 -9
  147. data/lib/woods/extractors/view_component_extractor.rb +28 -3
  148. data/lib/woods/extractors/view_engines/erb.rb +17 -3
  149. data/lib/woods/feedback/gap_detector.rb +9 -3
  150. data/lib/woods/feedback/store.rb +7 -1
  151. data/lib/woods/filename_utils.rb +29 -1
  152. data/lib/woods/flow_analysis/operation_extractor.rb +22 -10
  153. data/lib/woods/flow_assembler.rb +147 -26
  154. data/lib/woods/flow_document.rb +1 -0
  155. data/lib/woods/flow_precomputer.rb +175 -22
  156. data/lib/woods/gem_mapper.rb +285 -0
  157. data/lib/woods/generation.rb +185 -0
  158. data/lib/woods/git_command.rb +38 -0
  159. data/lib/woods/git_provenance.rb +16 -2
  160. data/lib/woods/graph_analyzer.rb +564 -87
  161. data/lib/woods/index_artifact.rb +93 -23
  162. data/lib/woods/mcp/bearer_auth.rb +102 -13
  163. data/lib/woods/mcp/bootstrap_state.rb +77 -0
  164. data/lib/woods/mcp/bootstrapper.rb +582 -77
  165. data/lib/woods/mcp/config_resolver.rb +66 -6
  166. data/lib/woods/mcp/errors.rb +60 -0
  167. data/lib/woods/mcp/index_reader.rb +836 -117
  168. data/lib/woods/mcp/index_reader_pinning.rb +78 -0
  169. data/lib/woods/mcp/origin_guard.rb +66 -7
  170. data/lib/woods/mcp/protocol_policy.rb +98 -0
  171. data/lib/woods/mcp/provider_probe.rb +45 -6
  172. data/lib/woods/mcp/renderers/markdown_renderer.rb +72 -4
  173. data/lib/woods/mcp/renderers/plain_renderer.rb +54 -6
  174. data/lib/woods/mcp/server.rb +898 -152
  175. data/lib/woods/mcp/tasks/extension.rb +196 -0
  176. data/lib/woods/mcp/tasks/request_capture.rb +45 -0
  177. data/lib/woods/mcp/tasks/store.rb +518 -0
  178. data/lib/woods/mcp/tool_contract.rb +171 -0
  179. data/lib/woods/mcp/tool_response_renderer.rb +7 -0
  180. data/lib/woods/model_name_cache.rb +19 -1
  181. data/lib/woods/notion/client.rb +132 -36
  182. data/lib/woods/notion/exporter.rb +456 -61
  183. data/lib/woods/notion/mappers/column_mapper.rb +34 -5
  184. data/lib/woods/notion/mappers/migration_mapper.rb +32 -8
  185. data/lib/woods/notion/mappers/model_mapper.rb +21 -6
  186. data/lib/woods/notion/mappers/shared.rb +45 -3
  187. data/lib/woods/notion/sync_manifest.rb +258 -0
  188. data/lib/woods/obsidian/errors.rb +6 -0
  189. data/lib/woods/obsidian/name_mapper.rb +40 -24
  190. data/lib/woods/obsidian/vault_exporter.rb +103 -36
  191. data/lib/woods/operator/pipeline_guard.rb +118 -21
  192. data/lib/woods/operator/status_reporter.rb +20 -3
  193. data/lib/woods/path_dispatcher.rb +276 -0
  194. data/lib/woods/payload_store.rb +236 -0
  195. data/lib/woods/published_index/edge_shaper.rb +61 -0
  196. data/lib/woods/published_index/generation_catalog.rb +72 -0
  197. data/lib/woods/published_index/typed_unit_reader.rb +48 -0
  198. data/lib/woods/published_index.rb +287 -0
  199. data/lib/woods/railtie.rb +69 -30
  200. data/lib/woods/railtie_support.rb +167 -0
  201. data/lib/woods/release.rb +12 -0
  202. data/lib/woods/reload_policy.rb +206 -0
  203. data/lib/woods/resilience/circuit_breaker.rb +47 -8
  204. data/lib/woods/resilience/index_validator.rb +296 -10
  205. data/lib/woods/resilience/retryable_provider.rb +71 -6
  206. data/lib/woods/resolved_config.rb +55 -11
  207. data/lib/woods/retrieval/context_assembler.rb +132 -40
  208. data/lib/woods/retrieval/query_classifier.rb +26 -8
  209. data/lib/woods/retrieval/ranker.rb +193 -28
  210. data/lib/woods/retrieval/search_executor.rb +206 -39
  211. data/lib/woods/retriever.rb +317 -71
  212. data/lib/woods/retry_after.rb +22 -2
  213. data/lib/woods/ruby_analyzer/class_analyzer.rb +10 -14
  214. data/lib/woods/ruby_analyzer/fqn_builder.rb +2 -0
  215. data/lib/woods/ruby_analyzer/mermaid_renderer.rb +14 -4
  216. data/lib/woods/ruby_analyzer/method_analyzer.rb +1 -1
  217. data/lib/woods/ruby_analyzer/trace_enricher.rb +3 -0
  218. data/lib/woods/ruby_analyzer.rb +21 -5
  219. data/lib/woods/session_tracer/file_store.rb +138 -19
  220. data/lib/woods/session_tracer/middleware.rb +1 -2
  221. data/lib/woods/session_tracer/redis_store.rb +122 -12
  222. data/lib/woods/session_tracer/session_flow_assembler.rb +57 -17
  223. data/lib/woods/session_tracer/session_flow_document.rb +56 -14
  224. data/lib/woods/session_tracer/solid_cache_coordination.rb +192 -0
  225. data/lib/woods/session_tracer/solid_cache_store.rb +560 -91
  226. data/lib/woods/session_tracer/store.rb +14 -1
  227. data/lib/woods/storage/metadata_store.rb +230 -26
  228. data/lib/woods/storage/pgvector.rb +180 -22
  229. data/lib/woods/storage/qdrant.rb +367 -41
  230. data/lib/woods/storage/snapshotter/metadata.rb +79 -16
  231. data/lib/woods/storage/snapshotter/vector.rb +128 -17
  232. data/lib/woods/storage/snapshotter.rb +23 -5
  233. data/lib/woods/storage/vector_store.rb +49 -8
  234. data/lib/woods/storage_identity.rb +28 -0
  235. data/lib/woods/tasks.rb +53 -2
  236. data/lib/woods/temporal/json_snapshot_store.rb +112 -42
  237. data/lib/woods/temporal/snapshot_store.rb +139 -42
  238. data/lib/woods/unblocked/client.rb +119 -17
  239. data/lib/woods/unblocked/document_builder.rb +34 -2
  240. data/lib/woods/unblocked/exporter.rb +63 -27
  241. data/lib/woods/unblocked/rate_limiter.rb +23 -9
  242. data/lib/woods/unblocked/sync_manifest.rb +16 -8
  243. data/lib/woods/update_check.rb +24 -1
  244. data/lib/woods/util/uuid5.rb +124 -0
  245. data/lib/woods/version.rb +1 -1
  246. data/lib/woods/watch/daemon.rb +1345 -0
  247. data/lib/woods/watch/listen_watcher.rb +81 -0
  248. data/lib/woods/watch/polling_watcher.rb +137 -0
  249. data/lib/woods/watch/status.rb +169 -0
  250. data/lib/woods/watch/tree_scan.rb +163 -0
  251. data/lib/woods/watch/watcher.rb +100 -0
  252. data/lib/woods.rb +138 -9
  253. data/plugin/.claude-plugin/plugin.json +18 -0
  254. data/plugin/hooks/hooks.json +29 -0
  255. data/plugin/hooks/woods-post-edit.sh +226 -0
  256. data/plugin/hooks/woods-session-start.sh +77 -0
  257. data/plugin/skills/woods-agent-enable/SKILL.md +51 -0
  258. data/plugin/skills/woods-diagnose/SKILL.md +75 -0
  259. data/plugin/skills/woods-investigate/SKILL.md +39 -0
  260. data/plugin/skills/woods-mcp-config/SKILL.md +101 -0
  261. data/plugin/skills/woods-setup/SKILL.md +99 -0
  262. metadata +134 -23
  263. data/lib/woods/console/adapters/cache_adapter.rb +0 -58
  264. data/lib/woods/console/adapters/good_job_adapter.rb +0 -33
  265. data/lib/woods/console/adapters/job_adapter.rb +0 -74
  266. data/lib/woods/console/adapters/sidekiq_adapter.rb +0 -33
  267. data/lib/woods/console/adapters/solid_queue_adapter.rb +0 -33
  268. data/lib/woods/console/bridge.rb +0 -210
  269. data/lib/woods/formatting/claude_adapter.rb +0 -98
  270. data/lib/woods/formatting/generic_adapter.rb +0 -56
  271. data/lib/woods/formatting/gpt_adapter.rb +0 -64
  272. data/lib/woods/notion/mapper.rb +0 -40
  273. data/lib/woods/observability/health_check.rb +0 -79
  274. data/lib/woods/observability/instrumentation.rb +0 -34
@@ -2,8 +2,12 @@
2
2
 
3
3
  require 'woods'
4
4
  require_relative 'client'
5
- require_relative 'mapper'
5
+ require_relative 'mappers/shared'
6
+ require_relative 'mappers/model_mapper'
7
+ require_relative 'mappers/column_mapper'
8
+ require_relative 'mappers/migration_mapper'
6
9
  require_relative 'rate_limiter'
10
+ require_relative 'sync_manifest'
7
11
 
8
12
  module Woods
9
13
  module Notion
@@ -13,18 +17,60 @@ module Woods
13
17
  # to Notion page properties, and pushes via the Notion API. All syncs are idempotent —
14
18
  # existing pages are updated, new pages are created.
15
19
  #
20
+ # Syncs are incremental (#207 / B-095): a {SyncManifest} persisted under
21
+ # the index directory records each page's Notion page_id and a content
22
+ # hash of its mapped properties, keyed by the page's qualified title
23
+ # (#149). An unchanged page is skipped with zero API calls; a changed
24
+ # page is PATCHed directly by its cached page_id (one call, no
25
+ # find-by-title query); only a manifest miss pays the full lookup /
26
+ # legacy-adoption / create path. A cold manifest (first run, deleted
27
+ # file) therefore degrades to exactly the pre-manifest behavior, plus
28
+ # manifest writes. Set +WOODS_NOTION_FORCE=1+ (or pass
29
+ # +force_full: true+) to ignore the manifest for one run.
30
+ #
16
31
  # @example
17
32
  # exporter = Exporter.new(index_dir: "tmp/woods")
18
33
  # stats = exporter.sync_all
19
- # # => { data_models: 10, columns: 45, errors: [] }
34
+ # # => { data_models: 10, columns: 45, skipped: 0, errors: [] }
20
35
  #
21
- class Exporter # rubocop:disable Metrics/ClassLength
36
+ # rubocop:disable-next Metrics/ClassLength
37
+ class Exporter
38
+ MAX_ERRORS = 100
39
+
40
+ # Manifest scope for the Data Models database.
41
+ SCOPE_DATA_MODELS = 'data_models'
42
+ # Manifest scope for the Columns database.
43
+ SCOPE_COLUMNS = 'columns'
44
+ # Manifest file name, stored under the index/output directory
45
+ # (mirrors Unblocked's +unblocked_sync_manifest.json+).
46
+ MANIFEST_FILENAME = 'notion_sync_manifest.json'
47
+ # Escape hatch: set to 1/true/yes to ignore the manifest for one run
48
+ # (the Notion counterpart of +UNBLOCKED_FORCE_FULL_SYNC+). Read here
49
+ # rather than in the rake task so the MCP notion tool and embedded
50
+ # callers honor it too.
51
+ FORCE_ENV_VAR = 'WOODS_NOTION_FORCE'
52
+
53
+ # Client errors that mean a cached page is gone (deleted or archived
54
+ # behind the manifest), so the sync should self-heal by recreating it.
55
+ # {Client} raises untyped +Woods::Error+ with the Notion status code
56
+ # baked into the message ("Notion API error 404: ..."), so this is a
57
+ # message-level match: 404/410 (page gone) or an archived-page
58
+ # complaint (Notion answers 400 "Can't update a page that is
59
+ # archived..." for trashed pages).
60
+ STALE_PAGE_ERROR = /\ANotion API error (?:404|410)\b|archiv/i
61
+
22
62
  # @param index_dir [String] Path to extraction output directory
23
63
  # @param config [Configuration] Woods configuration (default: global config)
24
64
  # @param client [Client, nil] Notion API client (auto-created from config if nil)
25
65
  # @param reader [Object, nil] IndexReader instance (auto-created from index_dir if nil)
66
+ # @param manifest [SyncManifest, nil] Sync manifest (auto-created under index_dir if nil)
67
+ # @param force_full [Boolean, nil] Ignore the manifest for this run —
68
+ # every page goes through the full find-by-title path (results are
69
+ # still recorded). Defaults to the {FORCE_ENV_VAR} env flag.
26
70
  # @raise [ConfigurationError] if notion_api_token is not configured
27
- def initialize(index_dir:, config: Woods.configuration, client: nil, reader: nil)
71
+ # rubocop:disable-next Metrics/ParameterLists -- injectable collaborators, mirrors Unblocked::Exporter
72
+ def initialize(index_dir:, config: Woods.configuration, client: nil, reader: nil,
73
+ manifest: nil, force_full: nil)
28
74
  # A non-blank NOTION_API_TOKEN overrides the configured token
29
75
  # (documented contract; also what the MCP notion_wired? gate keys on).
30
76
  # Resolve via the shared Woods.resolve_notion_token so the exporter,
@@ -37,61 +83,104 @@ module Woods
37
83
  @database_ids = config.notion_database_ids || {}
38
84
  @client = client || Client.new(api_token: api_token)
39
85
  @reader = reader || build_reader(index_dir)
86
+ @manifest = manifest || build_manifest(index_dir)
87
+ @force_full = force_full.nil? ? env_force? : force_full
40
88
  @page_id_cache = {}
41
89
  end
42
90
 
43
- # Sync all configured databases. Idempotent — safe to re-run.
91
+ # Sync all configured databases. Idempotent — safe to re-run; a re-run
92
+ # against an unchanged index issues zero API calls (see class docs).
44
93
  #
45
- # @return [Hash] { data_models: Integer, columns: Integer, errors: Array<String> }
94
+ # @return [Hash] { data_models: Integer, columns: Integer,
95
+ # skipped: Integer, errors: Array<String> }
46
96
  def sync_all
47
- model_stats = @database_ids[:data_models] ? sync_data_models : empty_stats
48
- column_stats = @database_ids[:columns] && @database_ids[:data_models] ? sync_columns : empty_stats
49
-
50
- all_errors = model_stats[:errors] + column_stats[:errors]
51
-
52
- {
53
- data_models: model_stats[:synced],
54
- columns: column_stats[:synced],
55
- errors: cap_errors(all_errors)
56
- }
97
+ with_pinned_index do
98
+ model_stats = @database_ids[:data_models] ? sync_data_models : empty_stats
99
+ warn_columns_without_data_models if @database_ids[:columns] && !@database_ids[:data_models]
100
+ column_stats = @database_ids[:columns] ? sync_columns : empty_stats
101
+
102
+ all_errors = model_stats[:errors] + column_stats[:errors]
103
+
104
+ {
105
+ data_models: model_stats[:synced],
106
+ columns: column_stats[:synced],
107
+ skipped: model_stats[:skipped] + column_stats[:skipped],
108
+ errors: cap_errors(all_errors)
109
+ }
110
+ end
57
111
  end
58
112
 
59
113
  # Sync model units to the Data Models Notion database.
60
114
  #
61
- # @return [Hash] { synced: Integer, errors: Array<String> }
115
+ # Pages are titled with the table name. When several models share one
116
+ # table (STI hierarchies, custom +self.table_name+ overlaps) a bare
117
+ # table-name title makes each model overwrite the others' pages (#149),
118
+ # so those titles are qualified as "<table> (<ModelName>)". Models with
119
+ # a unique table keep the bare title — no churn for the common case.
120
+ #
121
+ # Manifest entries whose title vanished from the current model set are
122
+ # pruned afterwards; the Notion pages themselves are left alone (no
123
+ # deletion path exists, and none is invented here).
124
+ #
125
+ # @return [Hash] { synced: Integer, skipped: Integer, errors: Array<String> }
62
126
  def sync_data_models
63
127
  database_id = @database_ids[:data_models]
64
128
  return empty_stats unless database_id
65
129
 
66
- migration_dates = load_migration_dates
67
- sync_units('model', database_id, 'Table Name') do |unit_data|
68
- properties = Mappers::ModelMapper.new.map(unit_data)
69
- enrich_with_migration_date(properties, migration_dates)
70
- properties
130
+ begin
131
+ migration_dates = load_migration_dates
132
+ shared_tables = shared_table_names
133
+ stats = sync_units('model', database_id, 'Table Name', SCOPE_DATA_MODELS) do |unit_data|
134
+ properties = Mappers::ModelMapper.new.map(unit_data)
135
+ # Enrichment reads the bare table name from the title — run it
136
+ # before any qualification rewrites the title.
137
+ enrich_with_migration_date(properties, migration_dates)
138
+ legacy_title = qualify_shared_table_title(properties, unit_data, shared_tables)
139
+ [properties, legacy_title]
140
+ end
141
+ @manifest.prune(SCOPE_DATA_MODELS, stats.delete(:current_keys))
142
+ stats
143
+ ensure
144
+ save_manifest
71
145
  end
72
146
  end
73
147
 
74
148
  # Sync column data to the Columns Notion database.
75
149
  #
76
- # @return [Hash] { synced: Integer, errors: Array<String> }
150
+ # Columns are synced **per physical table**, not per model. Column pages
151
+ # are titled "<table>.<column>", so two models sharing a table (STI, a
152
+ # shared +self.table_name=+) produced one page from two different
153
+ # payloads and PATCHed it back and forth on every run, forever (EXP-1).
154
+ # Grouping first means one page per physical column per run, with the
155
+ # Table relation listing every owning model and the validations unioned.
156
+ #
157
+ # Manifest entries whose qualified title vanished from the current
158
+ # column set are pruned afterwards; the Notion pages themselves are
159
+ # left alone (no deletion path exists, and none is invented here).
160
+ #
161
+ # @return [Hash] { synced: Integer, skipped: Integer, errors: Array<String> }
77
162
  def sync_columns
78
163
  database_id = @database_ids[:columns]
79
164
  return empty_stats unless database_id
80
165
 
81
- synced = 0
82
- errors = []
166
+ begin
167
+ totals = { synced: 0, skipped: 0, errors: [] }
168
+ current_keys = []
83
169
 
84
- each_model_unit do |entry, unit_data|
85
- synced_count, unit_errors = sync_model_columns(entry, unit_data, database_id)
86
- synced += synced_count
87
- errors.concat(unit_errors)
88
- end
170
+ column_groups.each do |group|
171
+ result = sync_table_columns(group, database_id, current_keys)
172
+ totals[:synced] += result[:synced]
173
+ totals[:skipped] += result[:skipped]
174
+ totals[:errors].concat(result[:errors])
175
+ end
89
176
 
90
- { synced: synced, errors: errors }
177
+ @manifest.prune(SCOPE_COLUMNS, current_keys)
178
+ totals
179
+ ensure
180
+ save_manifest
181
+ end
91
182
  end
92
183
 
93
- MAX_ERRORS = 100
94
-
95
184
  private
96
185
 
97
186
  # Sync all units of a type, yielding each for property mapping.
@@ -99,28 +188,39 @@ module Woods
99
188
  # @param type [String] Unit type to list
100
189
  # @param database_id [String] Notion database UUID
101
190
  # @param title_property [String] Name of the title property
102
- # @yield [Hash] Unit data hash, expects Notion properties hash back
103
- # @return [Hash] { synced: Integer, errors: Array<String> }
104
- def sync_units(type, database_id, title_property)
105
- synced = 0
106
- errors = []
191
+ # @param scope [String] Manifest scope for this database
192
+ # @yield [Hash] Unit data hash; expects back +[properties, legacy_title]+
193
+ # where +legacy_title+ (String, nil) is a pre-qualification title an
194
+ # already-synced page may still carry (see {#adopt_legacy_page})
195
+ # @return [Hash] { synced:, skipped:, errors:, current_keys: }
196
+ def sync_units(type, database_id, title_property, scope)
197
+ stats = { synced: 0, skipped: 0, errors: [], current_keys: [] }
107
198
 
108
199
  @reader.list_units(type: type).each do |entry|
109
200
  unit_data = @reader.find_unit(entry['identifier'])
110
201
  next unless unit_data
111
202
 
112
203
  begin
113
- properties = yield(unit_data)
204
+ properties, legacy_title = yield(unit_data)
114
205
  title_value = extract_title_text(properties[title_property])
115
- page_id = upsert_page(database_id: database_id, title_value: title_value, properties: properties)
206
+ stats[:current_keys] << title_value
207
+ legacy = legacy_title ? { title: legacy_title } : nil
208
+ status, page_id = sync_page(scope: scope, database_id: database_id, title_value: title_value,
209
+ properties: properties, legacy: legacy)
116
210
  @page_id_cache[entry['identifier']] = page_id
117
- synced += 1
211
+ stats[status] += 1
212
+ rescue AuthenticationError
213
+ # Not a per-unit failure: the token is wrong or the integration is
214
+ # not shared with this database, so every remaining call in the run
215
+ # would fail too — at Notion's 3 req/sec. Abort instead of spending
216
+ # the whole sync proving it.
217
+ raise
118
218
  rescue StandardError => e
119
- errors << "#{entry['identifier']}: #{e.message}"
219
+ stats[:errors] << "#{entry['identifier']}: #{e.message}"
120
220
  end
121
221
  end
122
222
 
123
- { synced: synced, errors: errors }
223
+ stats
124
224
  end
125
225
 
126
226
  # Iterate over loaded model units.
@@ -135,27 +235,189 @@ module Woods
135
235
  end
136
236
  end
137
237
 
138
- # Sync columns for a single model.
238
+ # Run a multi-read export body against one index generation.
239
+ #
240
+ # Every public IndexReader accessor self-refreshes when the published
241
+ # generation moves, and the reader assigns pinning responsibility to
242
+ # direct callers. Unpinned, a concurrent extraction mid-sync means the
243
+ # model list, the unit bodies and the page-id cache can straddle two
244
+ # generations (EXP-5) — a model present only in the newer one gets a nil
245
+ # parent page and a run of avoidable churn.
246
+ #
247
+ # Guarded by +respond_to?+: injected readers (specs, embedders) need not
248
+ # implement pinning.
249
+ #
250
+ # @yield the export body
251
+ # @return [Object] the block's value
252
+ def with_pinned_index(&block)
253
+ return yield unless @reader.respond_to?(:with_pinned_generation)
254
+
255
+ @reader.with_pinned_generation(&block)
256
+ end
257
+
258
+ # A columns-only configuration used to make {#sync_all} return all-zero
259
+ # stats in silence, which is indistinguishable from breakage (EXP-11).
260
+ # The sync runs; only the Table relation is unavailable, because its
261
+ # target pages live in the Data Models database nobody configured.
139
262
  #
140
- # @return [Array(Integer, Array<String>)] Count of synced columns and errors
141
- def sync_model_columns(entry, unit_data, database_id)
142
- parent_page_id = @page_id_cache[entry['identifier']]
143
- columns = unit_data.dig('metadata', 'columns') || []
144
- validations = unit_data.dig('metadata', 'validations') || []
263
+ # @return [void]
264
+ def warn_columns_without_data_models
265
+ warn 'woods: notion sync has no data_models database configured — ' \
266
+ 'syncing columns without a Table relation'
267
+ end
268
+
269
+ # One entry per physical table: every model that owns it, the union of
270
+ # their columns and validations, and the Data Models pages the Table
271
+ # relation should list.
272
+ #
273
+ # Parent page ids come from the page-id cache, which the manifest skip
274
+ # path populates too — so on a warm run the Table relation is
275
+ # byte-identical to the previous run and unchanged columns hash stable.
276
+ # Owners and page ids are sorted so the payload does not depend on the
277
+ # order the index happens to list models in.
278
+ #
279
+ # @return [Array<Hash>] { table_name:, owners:, columns:, validations:,
280
+ # parent_page_ids: }
281
+ def column_groups
282
+ groups = {}
283
+ each_model_unit { |entry, unit_data| absorb_into_column_group(groups, entry, unit_data) }
284
+ groups.each_value { |group| finalize_column_group(group) }
285
+ groups.values
286
+ end
287
+
288
+ # Fold one model's columns and validations into its table's group.
289
+ #
290
+ # @return [void]
291
+ def absorb_into_column_group(groups, entry, unit_data)
292
+ table_name = Mappers::ModelMapper.table_name_for(unit_data)
293
+ group = (groups[table_name] ||= { table_name: table_name, owners: [], columns: {}, validations: [] })
294
+ group[:owners] << entry['identifier']
295
+ (unit_data.dig('metadata', 'columns') || []).each { |column| group[:columns][column['name']] ||= column }
296
+ group[:validations].concat(unit_data.dig('metadata', 'validations') || [])
297
+ end
298
+
299
+ # Sort and dedupe a group so its payload does not depend on the order
300
+ # the index happens to list models in.
301
+ #
302
+ # @return [void]
303
+ def finalize_column_group(group)
304
+ group[:owners] = group[:owners].uniq.sort
305
+ group[:validations] = group[:validations].uniq
306
+ group[:parent_page_ids] = group[:owners].filter_map { |owner| @page_id_cache[owner] }.uniq
307
+ end
308
+
309
+ # Sync the columns of one physical table.
310
+ #
311
+ # Column pages are titled "<table>.<column>" (#149). Every model shares
312
+ # id/created_at/updated_at, and upserting by the bare column name found
313
+ # the previous model's page by title equality across the whole Columns
314
+ # database — churning it down to one page per distinct column name.
315
+ # The table qualifier comes from {Mappers::ModelMapper.table_name_for},
316
+ # the same value the Table relation's target page is titled with.
317
+ #
318
+ # @param group [Hash] One {#column_groups} entry
319
+ # @param current_keys [Array<String>] Sink for this run's column titles
320
+ # @return [Hash] { synced: Integer, skipped: Integer, errors: Array<String> }
321
+ def sync_table_columns(group, database_id, current_keys)
145
322
  mapper = Mappers::ColumnMapper.new
146
- synced = 0
147
- errors = []
148
-
149
- columns.each do |column|
150
- properties = mapper.map(column, model_identifier: entry['identifier'],
151
- validations: validations, parent_page_id: parent_page_id)
152
- upsert_page(database_id: database_id, title_value: column['name'], properties: properties)
153
- synced += 1
323
+ stats = { synced: 0, skipped: 0, errors: [] }
324
+ owners = group[:owners].join(', ')
325
+
326
+ group[:columns].each_value do |column|
327
+ stats[sync_one_column(mapper, group, column, database_id, current_keys)] += 1
328
+ rescue AuthenticationError
329
+ raise # see sync_units an auth failure dooms the whole run
154
330
  rescue StandardError => e
155
- errors << "#{entry['identifier']}.#{column['name']}: #{e.message}"
331
+ stats[:errors] << "#{owners}.#{column['name']}: #{e.message}"
332
+ end
333
+
334
+ stats
335
+ end
336
+
337
+ # Sync one physical column's page.
338
+ #
339
+ # @return [Symbol] :synced or :skipped
340
+ def sync_one_column(mapper, group, column, database_id, current_keys)
341
+ properties = mapper.map(column, model_identifier: group[:owners].first, table_name: group[:table_name],
342
+ validations: group[:validations],
343
+ parent_page_ids: group[:parent_page_ids])
344
+ title_value = extract_title_text(properties['Column Name'])
345
+ current_keys << title_value
346
+ status, = sync_page(scope: SCOPE_COLUMNS, database_id: database_id, title_value: title_value,
347
+ properties: properties,
348
+ legacy: column_legacy_descriptor(column, group[:parent_page_ids].first))
349
+ status
350
+ end
351
+
352
+ # Sync one page through the manifest. Three tiers, cheapest first:
353
+ # manifest hit with an unchanged hash — skip, zero API calls; hit with
354
+ # a changed hash — PATCH the cached page_id directly (one call, with
355
+ # gone-page self-heal); miss — the full #149 path ({#upsert_page}:
356
+ # find by qualified title, legacy adoption, create/update). Every
357
+ # non-skip outcome records page_id + content hash for the next run.
358
+ # +force_full+ bypasses the manifest reads entirely (tier three for
359
+ # everything) but still records.
360
+ #
361
+ # @param scope [String] Manifest scope
362
+ # @param database_id [String] Notion database UUID
363
+ # @param title_value [String] Qualified title — the manifest key
364
+ # @param properties [Hash] Full property payload to write
365
+ # @param legacy [Hash, nil] Legacy descriptor for {#upsert_page}
366
+ # @return [Array(Symbol, String)] [:skipped or :synced, page id]
367
+ def sync_page(scope:, database_id:, title_value:, properties:, legacy: nil)
368
+ content_hash = SyncManifest.content_hash(properties)
369
+
370
+ unless @force_full
371
+ if @manifest.unchanged?(scope, title_value, content_hash)
372
+ return [:skipped, @manifest.page_id_for(scope, title_value)]
373
+ end
374
+
375
+ cached_id = @manifest.page_id_for(scope, title_value)
376
+ if cached_id && (page_id = update_cached_page(scope, title_value, cached_id, properties))
377
+ @manifest.record(scope: scope, key: title_value, hash: content_hash, page_id: page_id)
378
+ return [:synced, page_id]
379
+ end
156
380
  end
157
381
 
158
- [synced, errors]
382
+ page_id = upsert_page(database_id: database_id, title_value: title_value,
383
+ properties: properties, legacy: legacy)
384
+ @manifest.record(scope: scope, key: title_value, hash: content_hash, page_id: page_id)
385
+ [:synced, page_id]
386
+ end
387
+
388
+ # PATCH a page by its manifest-cached id — the one-call path for
389
+ # changed content. When Notion reports the page gone ({STALE_PAGE_ERROR}:
390
+ # deleted or archived behind the cache), warn once, drop the manifest
391
+ # entry, and return nil so the caller falls through to the create path
392
+ # (self-heal). Any other API failure propagates to the per-unit rescue
393
+ # so it lands in the error list with the entry retained for a retry.
394
+ #
395
+ # @return [String, nil] the page id on success, nil when the cached page is gone
396
+ def update_cached_page(scope, key, page_id, properties)
397
+ @client.update_page(page_id: page_id, properties: properties)
398
+ page_id
399
+ rescue Woods::Error => e
400
+ raise unless e.message.match?(STALE_PAGE_ERROR)
401
+
402
+ warn "woods: notion sync cached page #{page_id} for #{key.inspect} is gone (#{e.message}) — recreating"
403
+ @manifest.forget(scope, key)
404
+ nil
405
+ end
406
+
407
+ # Describe the pre-#149 page a column may still be stored under: the
408
+ # bare column name, verified against this model's Table relation when
409
+ # the parent Data Models page is known.
410
+ #
411
+ # @param column [Hash]
412
+ # @param parent_page_id [String, nil]
413
+ # @return [Hash] Legacy descriptor for {#upsert_page}
414
+ def column_legacy_descriptor(column, parent_page_id)
415
+ legacy = { title: column['name'].to_s }
416
+ if parent_page_id
417
+ legacy[:relation_property] = 'Table'
418
+ legacy[:relation_page_id] = parent_page_id
419
+ end
420
+ legacy
159
421
  end
160
422
 
161
423
  # Enrich model properties with migration date if available.
@@ -182,9 +444,22 @@ module Woods
182
444
 
183
445
  # Upsert a Notion page: find by title, update if exists, create if not.
184
446
  #
447
+ # When the qualified-title lookup misses and a +legacy+ descriptor is
448
+ # given, a page still carrying the pre-qualification title may be
449
+ # adopted instead of creating a duplicate (see {#adopt_legacy_page}).
450
+ # Updating the adopted page rewrites its title to the qualified form,
451
+ # so the legacy lookup is a one-time migration cost per page.
452
+ #
453
+ # @param database_id [String] Notion database UUID
454
+ # @param title_value [String] Title to find the page by
455
+ # @param properties [Hash] Full property payload to write
456
+ # @param legacy [Hash, nil] { title:, relation_property:, relation_page_id: }
185
457
  # @return [String] Notion page ID
186
- def upsert_page(database_id:, title_value:, properties:)
458
+ def upsert_page(database_id:, title_value:, properties:, legacy: nil)
187
459
  existing = @client.find_page_by_title(database_id: database_id, title: title_value)
460
+ if existing.nil? && legacy && legacy[:title] != title_value
461
+ existing = adopt_legacy_page(database_id: database_id, legacy: legacy, new_title: title_value)
462
+ end
188
463
 
189
464
  if existing
190
465
  @client.update_page(page_id: existing['id'], properties: properties)
@@ -195,9 +470,129 @@ module Woods
195
470
  end
196
471
  end
197
472
 
473
+ # Locate a page still titled with the pre-#149 (unqualified) title so
474
+ # the sync can adopt it — update it in place, which also rewrites its
475
+ # title to the qualified form — rather than strand it next to a fresh
476
+ # duplicate forever.
477
+ #
478
+ # Adoption is deliberately conservative:
479
+ # - With a relation qualifier (Columns), the title-equality filter is
480
+ # AND-ed with a `relation contains` filter on the parent Data Models
481
+ # page, so only a legacy page actually belonging to this model can be
482
+ # adopted. Multiple matches are same-parent duplicates; the first is
483
+ # adopted with a warning.
484
+ # - Without one (Data Models), the page is adopted only when exactly one
485
+ # match exists; an ambiguous set is left alone with a warning and a
486
+ # fresh qualified page is created instead.
487
+ #
488
+ # @param database_id [String] Notion database UUID
489
+ # @param legacy [Hash] { title:, relation_property:, relation_page_id: }
490
+ # @param new_title [String] Qualified title the adopted page will get
491
+ # @return [Hash, nil] Adoptable page, or nil to create fresh
492
+ def adopt_legacy_page(database_id:, legacy:, new_title:)
493
+ results = query_legacy_pages(database_id, legacy)
494
+ return nil if results.empty?
495
+
496
+ if results.size > 1
497
+ return warn_ambiguous_legacy(legacy, results) unless legacy[:relation_page_id]
498
+
499
+ warn "woods: notion sync found #{results.size} legacy pages titled #{legacy[:title].inspect} " \
500
+ "for one Table relation; adopting the first (#{results.first['id']})"
501
+ end
502
+
503
+ warn "woods: notion sync adopting legacy page #{results.first['id']} " \
504
+ "(#{legacy[:title].inspect} -> #{new_title.inspect})"
505
+ results.first
506
+ end
507
+
508
+ # @param legacy [Hash]
509
+ # @param results [Array<Hash>]
510
+ # @return [nil]
511
+ def warn_ambiguous_legacy(legacy, results)
512
+ warn "woods: notion sync found #{results.size} pages sharing the legacy title " \
513
+ "#{legacy[:title].inspect} and cannot tell them apart; creating a qualified page " \
514
+ 'instead — clean up the legacy pages manually'
515
+ nil
516
+ end
517
+
518
+ # Query pages carrying the legacy title, narrowed by the parent relation
519
+ # when the descriptor names one. This is the only extra API call the
520
+ # qualified-title scheme introduces, and it runs solely on the
521
+ # qualified-lookup miss path.
522
+ #
523
+ # @param database_id [String]
524
+ # @param legacy [Hash]
525
+ # @return [Array<Hash>] Matching pages
526
+ def query_legacy_pages(database_id, legacy)
527
+ filter = { property: 'title', title: { equals: legacy[:title] } }
528
+ if legacy[:relation_property] && legacy[:relation_page_id]
529
+ filter = { and: [filter, { property: legacy[:relation_property],
530
+ relation: { contains: legacy[:relation_page_id] } }] }
531
+ end
532
+
533
+ response = @client.query_database(database_id: database_id, filter: filter)
534
+ response['results'] || []
535
+ end
536
+
537
+ # Table names claimed by more than one model unit in the index — STI
538
+ # hierarchies, or models sharing a table via +self.table_name=+.
539
+ #
540
+ # @return [Array<String>]
541
+ def shared_table_names
542
+ counts = Hash.new(0)
543
+ each_model_unit do |_entry, unit_data|
544
+ counts[Mappers::ModelMapper.table_name_for(unit_data)] += 1
545
+ end
546
+ counts.select { |_table, count| count > 1 }.keys
547
+ end
548
+
549
+ # Rewrite the Data Models title to "<table> (<ModelName>)" for a model
550
+ # whose table name is shared with another model, so the two stop
551
+ # overwriting each other's pages (#149).
552
+ #
553
+ # @param properties [Hash] Mapped Notion properties (mutated)
554
+ # @param unit_data [Hash]
555
+ # @param shared_tables [Array<String>]
556
+ # @return [String, nil] The bare table name as the legacy title when
557
+ # qualification was applied, nil otherwise
558
+ def qualify_shared_table_title(properties, unit_data, shared_tables)
559
+ table_name = Mappers::ModelMapper.table_name_for(unit_data)
560
+ return nil unless shared_tables.include?(table_name)
561
+
562
+ qualified = "#{table_name} (#{unit_data['identifier']})"
563
+ properties['Table Name'] = { title: [{ text: { content: qualified } }] }
564
+ table_name
565
+ end
566
+
567
+ # Persist the manifest, downgrading failures to a warning: a lost
568
+ # manifest only costs a full re-check next run, which must not turn an
569
+ # otherwise-successful sync into a crash (called from ensure blocks,
570
+ # where a raise would also mask any in-flight exception).
571
+ #
572
+ # @return [void]
573
+ def save_manifest
574
+ @manifest.save
575
+ rescue StandardError => e
576
+ warn "woods: notion sync manifest not persisted (#{e.message}) — next run re-checks every page"
577
+ end
578
+
579
+ # @param index_dir [String]
580
+ # @return [SyncManifest]
581
+ def build_manifest(index_dir)
582
+ SyncManifest.new(path: File.join(index_dir, MANIFEST_FILENAME), database_ids: @database_ids)
583
+ end
584
+
585
+ # Truthy set mirrors the rake tasks' env_flag convention, so
586
+ # WOODS_NOTION_FORCE=false / =0 disables rather than silently enabling.
587
+ #
588
+ # @return [Boolean]
589
+ def env_force?
590
+ %w[1 true yes].include?(ENV.fetch(FORCE_ENV_VAR, '').strip.downcase)
591
+ end
592
+
198
593
  # @return [Hash]
199
594
  def empty_stats
200
- { synced: 0, errors: [] }
595
+ { synced: 0, skipped: 0, errors: [] }
201
596
  end
202
597
 
203
598
  # Cap errors to prevent unbounded memory growth.