opensearch-sugar 1.0.0

data/.agents/skills/diataxis/scripts/generate_index.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""
+Script to generate index.md for Diátaxis-organized documentation.
+
+Scans the specified docs root (default .docs) for subfolders: tutorials, how-to, reference, explanation.
+Recursively renders nested sub-category folders (e.g., reference/ralph/, how-to/copilot/cli/).
+Extracts titles from .md files and generates an index.md file with links organized by category
+and sub-category.
+"""
+
+import os
+import sys
+from pathlib import Path
+
+
+def get_title(filepath):
+    """Extract the title from the first line of a markdown file."""
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            first_line = f.readline().strip()
+            if first_line.startswith('# '):
+                return first_line[2:].strip()
+            else:
+                return os.path.basename(filepath).replace('.md', '').replace('-', ' ').title()
+    except Exception as e:
+        print(f"Error reading {filepath}: {e}")
+        return os.path.basename(filepath).replace('.md', '').replace('-', ' ').title()
+
+
+def has_visible_content(directory):
+    """Return True when the directory contains visible files or subdirectories."""
+    for entry in directory.iterdir():
+        if entry.name == '.gitkeep':
+            return True
+        if not entry.name.startswith('.'):
+            return True
+    return False
+
+
+def render_directory(directory, docs_root, heading_level):
+    """Render a directory and any nested subdirectories recursively."""
+    direct_files = sorted(
+        f for f in directory.iterdir()
+        if f.is_file() and f.suffix == '.md'
+    )
+    child_dirs = sorted(
+        d for d in directory.iterdir()
+        if d.is_dir() and not d.name.startswith('.')
+    )
+    visible_children = [d for d in child_dirs if has_visible_content(d)]
+    has_keep_marker = any(entry.name == '.gitkeep' for entry in directory.iterdir())
+
+    if not direct_files and not visible_children and not has_keep_marker:
+        return ""
+
+    heading = directory.name.replace('-', ' ').title()
+    content = f"{'#' * heading_level} {heading}\n\n"
+
+    for file in direct_files:
+        title = get_title(str(file))
+        link = f"{directory.relative_to(docs_root).as_posix()}/{file.name}"
+        content += f"- [{title}]({link})\n"
+
+    if direct_files and visible_children:
+        content += "\n"
+
+    if not direct_files and not visible_children:
+        content += "- _No documents yet._\n\n"
+
+    for child_dir in visible_children:
+        child_content = render_directory(child_dir, docs_root, heading_level + 1)
+        if child_content:
+            content += child_content
+
+    return content
+
+
+def main(docs_root='.docs'):
+    """Generate the index.md file."""
+    categories = {
+        'tutorials': 'Tutorials',
+        'how-to': 'How-to Guides',
+        'reference': 'Reference',
+        'explanation': 'Explanation'
+    }
+
+    index_content = """# Copilot Workspace Documentation Index
+
+This index links Diátaxis-organized documentation for the workspace.
+
+"""
+
+    docs_root_path = Path(docs_root)
+
+    for folder, section in categories.items():
+        cat_path = docs_root_path / folder
+        if not cat_path.is_dir():
+            continue
+
+        root_files = sorted(
+            f for f in cat_path.iterdir()
+            if f.is_file() and f.suffix == '.md'
+        )
+        subdirs = sorted(
+            d for d in cat_path.iterdir()
+            if d.is_dir() and not d.name.startswith('.')
+        )
+        visible_subdirs = [d for d in subdirs if has_visible_content(d)]
+
+        if not root_files and not visible_subdirs:
+            continue
+
+        index_content += f"## {section}\n\n"
+
+        for file in root_files:
+            title = get_title(str(file))
+            link = f"{folder}/{file.name}"
+            index_content += f"- [{title}]({link})\n"
+
+        if root_files and visible_subdirs:
+            index_content += "\n"
+
+        for subdir in visible_subdirs:
+            index_content += render_directory(subdir, docs_root_path, 3)
+            if not index_content.endswith("\n\n"):
+                index_content += "\n"
+
+        if not root_files:
+            index_content += "\n"
+
+    index_path = os.path.join(docs_root, 'index.md')
+    index_content = index_content.rstrip() + "\n"
+    with open(index_path, 'w', encoding='utf-8') as f:
+        f.write(index_content)
+    print(f"Generated {index_path}")
+
+if __name__ == '__main__':
+    docs_root = sys.argv[1] if len(sys.argv) > 1 else '.docs'
+    main(docs_root)

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.standard.yml

@@ -0,0 +1,3 @@
+# For available configuration options, see:
+#   https://github.com/standardrb/standard
+ruby_version: 3.1

data/AGENTS.md

@@ -0,0 +1,120 @@
+# opensearch-sugar — Agent Instructions
+
+## What this project is
+
+`opensearch-sugar` is a Ruby gem that wraps the [`opensearch-ruby`](https://github.com/opensearch-project/opensearch-ruby)
+client with a more convenient, object-oriented API. It provides three main classes:
+
+- **`OpenSearch::Sugar::Client`** — connects to an OpenSearch cluster; adds index management,
+  settings/mappings helpers, and delegated access to all underlying `OpenSearch::Client` methods
+- **`OpenSearch::Sugar::Index`** — represents a single index with methods for CRUD on documents,
+  settings, mappings, aliases, and text analysis
+- **`OpenSearch::Sugar::Models`** — manages ML models via the OpenSearch ML Commons plugin
+  (register, deploy, list, delete, and build embedding pipelines)
+
+This is a plain Ruby gem. It has no Rails dependency and must not use Rails-specific gems or patterns.
+
+## File structure
+
+```
+lib/
+  opensearch/
+    sugar.rb                  # Top-level require; defines OpenSearch::Sugar module
+    sugar/
+      client.rb               # OpenSearch::Sugar::Client (SimpleDelegator wrapper)
+      index.rb                # OpenSearch::Sugar::Index
+      index/                  # Index sub-components
+      models.rb               # OpenSearch::Sugar::Models
+      version.rb              # OpenSearch::Sugar::VERSION
+
+sig/
+  opensearch/
+    sugar.rbs                 # RBS type signatures for the public API
+
+spec/
+  spec_helper.rb
+  opensearch_integration_spec.rb
+  env.testing                 # dotenv file for the test cluster
+
+docs/
+  REFERENCE.md                # Complete API reference
+  HOWTO.md                    # Problem-solving recipes
+  TUTORIAL.md                 # Step-by-step learning guide
+  EXPLANATION.md              # Conceptual background
+  DELEGATED_METHODS_ANALYSIS.md
+
+adrs/                         # Architecture Decision Records
+```
+
+## Code style
+
+- **Formatter**: Ruby `standard` gem with default settings. Config at `.standard.yml`. Do not suppress rules without a comment explaining why.
+- **Line length**: 100 characters (`.editorconfig`).
+
+## Test stack
+
+Tests are written with **RSpec** and live in `spec/`. They are **integration tests** that
+require a live OpenSearch node; there is no mocking layer. Spin up a local cluster first:
+
+```bash
+docker compose up -d
+bundle exec rspec
+```
+
+Environment variables for the test cluster are loaded from `spec/env.testing` via `dotenv`.
+There is currently no unit-test suite; if you add pure-Ruby logic to `lib/` that can be
+tested in isolation, add unit specs alongside it.
+
+## Naming conventions
+
+- **Top-level namespace**: `OpenSearch::Sugar`
+- **Classes**: `OpenSearch::Sugar::Client`, `OpenSearch::Sugar::Index`, `OpenSearch::Sugar::Models`
+- **Error base class**: `OpenSearch::Sugar::Error < StandardError`
+- **Source files**: `snake_case` under `lib/opensearch/sugar/` (e.g. `client.rb`, `models.rb`)
+- **Spec files**: mirror source path with `_spec.rb` suffix (e.g. `spec/opensearch/sugar/client_spec.rb`)
+- **RBS files**: `snake_case` under `sig/opensearch/sugar/` with `.rbs` extension
+- **Script/rake filenames**: `snake_case` (Ruby: `.rb`, bash: `.sh`)
+
+## Type signatures
+
+RBS type signatures for the public API live in `sig/`. When adding or changing public methods,
+update the corresponding `.rbs` file to keep signatures in sync. Type-check with `steep check`
+if Steep is configured, or validate signatures with `rbs validate`.
+
+## Documentation
+
+- General documentation conventions follow `/Users/dueberb/devel/ai/agent_resources/external_documentation.md`.
+- Project metadata conventions (versioning, changelog, release process, CONTRIBUTING, SECURITY) follow
+  `/Users/dueberb/devel/ai/agent_resources/project_metadata.md`.
+- **Diagrams**: use Mermaid in both ADRs and `docs/` files.
+
+## Ruby idioms and best practices
+
+- Use modern Ruby syntax and idioms (post-3.4) unless there is a compelling reason not to.
+- Avoid `_1`-style block variables for readability.
+- Use exceptions for error handling rather than Result objects or similar patterns, unless there is a compelling reason to do otherwise.
+- Don't use Rails-specific gems or patterns, as this is a general-purpose Ruby gem that does not assume a Rails environment.
+- Use YARD for API documentation, with clear comments on all public methods and classes. Include links to relevant OpenSearch documentation where applicable.
+
+## ADRs
+
+All design decisions are recorded in `adrs/`.  Use the template in `adrs/ADR-000-template.md` for consistency.
+When making a non-trivial architectural
+choice, check the ADRs first. If a decision conflicts with an ADR, raise it explicitly
+rather than quietly working around it.
+
+## Where things go
+
+| What | Where | Notes |
+|---|---|---|
+| Public API documentation | `docs/REFERENCE.md` | Method signatures, parameters, return values |
+| How-to recipes | `docs/HOWTO.md` | Task-oriented guides for common problems |
+| Tutorial content | `docs/TUTORIAL.md` | Step-by-step learning walkthroughs |
+| Conceptual background | `docs/EXPLANATION.md` | Why things work the way they do |
+| Architecture decisions | `adrs/` | Use `ADR-NNN-title.md` format; scaffold with the `adr-create` skill |
+| Agent/AI working plans | project root | Uppercase name, e.g. `Increase_Coverage.md`; these are working documents, not user-facing docs |
+| Gem source code | `lib/opensearch/sugar/` | `snake_case.rb`; mirror namespace in directory structure |
+| Spec files | `spec/opensearch/sugar/` | Mirror source path with `_spec.rb` suffix |
+| RBS type signatures | `sig/opensearch/sugar/` | Mirror source path with `.rbs` suffix |
+| Rake tasks | `Rakefile` | Keep tasks focused; add a `desc` to every task |
+| Gem dependencies | `opensearch-sugar.gemspec` | Runtime deps via `add_dependency`; dev deps via `add_development_dependency` |

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2025-06-19
+
+- Initial release

data/Dockerfile.opensearch

@@ -0,0 +1,4 @@
+FROM opensearchproject/opensearch:latest AS opensearch
+
+RUN bin/opensearch-plugin install analysis-icu
+

data/Increase_Coverage.md

@@ -0,0 +1,311 @@
+# Plan: Increase Test Coverage - Phase 1 Quick Wins
+
+## Current Status
+
+- **Overall Coverage:** 78.2% (165/211 lines)
+- **By File:**
+  - `opensearch/sugar/models.rb`: 25.9% (14/54 lines) - Intentionally excluded (`:models` tag)
+  - `opensearch/sugar/client.rb`: 93.8% (60/64 lines) - 4 uncovered lines
+  - `opensearch/sugar/index.rb`: 97.5% (78/80 lines) - 2 uncovered lines
+
+## Objective
+
+Increase test coverage from **78.2%** to **~82%** by adding tests for uncovered code paths in the Client and Index classes. This focuses on achievable, high-value improvements without requiring ML plugin setup or complex edge case testing.
+
+---
+
+## Phase 1: Quick Wins (Target: +3-4% coverage)
+
+### 1. Add Unwrapped Settings Format Test
+
+**File:** `spec/opensearch/sugar/client/settings_spec.rb`  
+**Coverage target:** `client.rb:181`  
+**Issue:** Tests only use wrapped format `{ settings: { ... } }`. Need to test unwrapped format `{ analysis: { ... } }`.
+
+**Implementation:**
+
+Add a new describe block to test unwrapped settings format:
+
+```ruby
+describe "#update_settings with unwrapped format" do
+  let(:index_name) { "sugar_test_#{SecureRandom.hex(6)}" }
+  let(:analyzer_settings_unwrapped) do
+    {
+      analysis: {
+        analyzer: {
+          test_unwrapped: {
+            type: "custom",
+            tokenizer: "standard",
+            filter: ["lowercase"]
+          }
+        }
+      }
+    }
+  end
+
+  before { client.create_index!(index_name) }
+  after { client.delete_index!(index_name) rescue nil }
+
+  it "accepts settings without wrapping in 'settings' key" do
+    expect { client.update_settings(analyzer_settings_unwrapped, index_name) }.not_to raise_error
+  end
+
+  it "applies the unwrapped settings correctly" do
+    client.update_settings(analyzer_settings_unwrapped, index_name)
+    index = client.open_index(index_name)
+    expect(index.all_available_analyzers).to include("test_unwrapped")
+  end
+end
+```
+
+---
+
+### 2. Add Unwrapped Mappings Format Test
+
+**File:** `spec/opensearch/sugar/index/mappings_spec.rb`  
+**Coverage target:** `client.rb:218`  
+**Issue:** Tests only use wrapped format `{ mappings: { ... } }`. Need to test unwrapped format `{ properties: { ... } }`.
+
+**Implementation:**
+
+Add a new describe block to test unwrapped mappings format:
+
+```ruby
+describe "#update_mappings with unwrapped format" do
+  let(:new_mappings_unwrapped) do
+    {
+      properties: {
+        unwrapped_title: { type: "text" },
+        unwrapped_count: { type: "integer" }
+      }
+    }
+  end
+
+  it "accepts mappings without wrapping in 'mappings' key" do
+    expect { index.update_mappings(new_mappings_unwrapped) }.not_to raise_error
+  end
+
+  it "makes the new fields visible in mappings after the update" do
+    index.update_mappings(new_mappings_unwrapped)
+    props = index.mappings.dig(index_name, "mappings", "properties")
+    expect(props).to include("unwrapped_title", "unwrapped_count")
+  end
+end
+```
+
+---
+
+### 3. Add Synonym Analyzer Test (Client)
+
+**File:** `spec/opensearch/sugar/client/analyzer_spec.rb`  
+**Coverage target:** `client.rb:280`  
+**Issue:** Missing test for tokens at the same position (synonyms, word decomposition).
+
+**Implementation:**
+
+Add a test case that uses a synonym filter to generate same-position tokens:
+
+```ruby
+describe "#test_analyzer_by_definition with synonym filter" do
+  it "returns tokens at the same position as arrays when synonyms are used" do
+    tokens = client.test_analyzer_by_definition(
+      text: "quick brown fox",
+      tokenizer: "standard",
+      filter: [
+        {
+          type: "synonym",
+          synonyms: ["quick, fast"]
+        }
+      ]
+    )
+    
+    # Should return: [["quick", "fast"], "brown", "fox"] or similar
+    # The exact format depends on synonym expansion behavior
+    expect(tokens).to be_an(Array)
+    expect(tokens.first).to be_an(Array).or be_a(String)
+  end
+
+  it "handles multi-word synonyms that produce same-position tokens" do
+    tokens = client.test_analyzer_by_definition(
+      text: "wi-fi",
+      tokenizer: "standard",
+      filter: [
+        {
+          type: "word_delimiter",
+          split_on_case_change: false,
+          split_on_numerics: false
+        }
+      ]
+    )
+    
+    expect(tokens).to be_an(Array)
+  end
+end
+```
+
+---
+
+### 4. Add Synonym Analyzer Test (Index)
+
+**File:** `spec/opensearch/sugar/index/analyzer_spec.rb`  
+**Coverage target:** `index.rb:210`  
+**Issue:** Missing test for tokens at the same position (synonyms, word decomposition).
+
+**Implementation:**
+
+Add a test case that creates an analyzer with synonyms:
+
+```ruby
+describe "#test_analyzer_by_name with synonym filter" do
+  let(:synonym_analyzer_settings) do
+    {
+      settings: {
+        analysis: {
+          filter: {
+            synonym_filter: {
+              type: "synonym",
+              synonyms: ["quick, fast", "big, large"]
+            }
+          },
+          analyzer: {
+            synonym_analyzer: {
+              type: "custom",
+              tokenizer: "standard",
+              filter: ["lowercase", "synonym_filter"]
+            }
+          }
+        }
+      }
+    }
+  end
+
+  let(:index_with_synonyms) do
+    idx = OpenSearch::Sugar::Index.create(client: client, name: index_name)
+    idx.update_settings(synonym_analyzer_settings)
+    idx
+  end
+
+  before { index_with_synonyms }
+
+  it "returns tokens at the same position as arrays when synonyms expand" do
+    tokens = index_with_synonyms.test_analyzer_by_name(
+      analyzer: "synonym_analyzer",
+      text: "quick brown fox"
+    )
+    
+    # Should include both "quick" and "fast" at position 0
+    expect(tokens).to be_an(Array)
+    flattened = tokens.flatten
+    expect(flattened).to include("quick").or include("fast")
+  end
+end
+```
+
+---
+
+## Expected Outcome
+
+After implementing Phase 1:
+- **Estimated coverage:** ~82% (up from 78.2%)
+- **Lines covered:** +8-10 lines across Client and Index classes
+- **Test execution time:** Minimal increase (~2-3 seconds)
+- **Files modified:** 3 spec files
+
+---
+
+## What We're NOT Doing (And Why)
+
+### Phase 2: Model Tests - Skipped for Now
+
+**Why we're skipping:**
+- Model tests exist but are excluded by default (`:models` tag in `spec_helper.rb:34`)
+- Requires ML Commons plugin setup in the test cluster
+- Tests are marked `:slow` and take significantly longer to run
+- The 40 uncovered lines in `models.rb` represent 25.9% coverage, but this is a separate testing concern
+- Model functionality may require a different testing strategy (mocking, VCR, or dedicated ML test environment)
+
+**Current model test files (not running):**
+- `spec/opensearch/sugar/models/registration_spec.rb` (38 lines)
+- `spec/opensearch/sugar/models/lookup_spec.rb` (57 lines)
+- `spec/opensearch/sugar/models/lifecycle_spec.rb` (40 lines)
+- `spec/opensearch/sugar/models/pipeline_spec.rb` (50 lines)
+
+**When to revisit:**
+- When ML Commons plugin is consistently available in CI
+- When we decide on a testing strategy for slow/external-dependency tests
+- When we want to push coverage above 85%
+
+---
+
+### Phase 3: Edge Cases - Skipped for Now
+
+**Why we're skipping:**
+
+1. **Double-failure recovery (`client.rb:297`)**
+   - Requires forcing two consecutive failures (update fails AND reopen fails)
+   - Complex to test reliably in integration tests
+   - Defensive code with low user impact
+   - ROI: Low - edge case that's rarely hit in practice
+
+2. **Default index body (`index.rb:331`)**
+   - Private method, already indirectly tested
+   - All current tests explicitly provide settings
+   - SimpleCov may not track private method calls correctly
+   - ROI: Very low - method IS executed, just not explicitly tested
+
+**When to revisit:**
+- When aiming for 90%+ coverage
+- When edge case bugs are discovered
+- When refactoring makes these paths more testable
+
+---
+
+## Implementation Checklist
+
+- [x] Add unwrapped settings format test to `spec/opensearch/sugar/client/settings_spec.rb`
+- [x] Add unwrapped mappings format test to `spec/opensearch/sugar/index/mappings_spec.rb`
+- [x] Add synonym analyzer test to `spec/opensearch/sugar/client/analyzer_spec.rb`
+- [x] Add synonym analyzer test to `spec/opensearch/sugar/index/analyzer_spec.rb`
+- [x] Run coverage: `bundle exec rake coverage`
+- [x] Verify new coverage percentage — **actual result: 80.1%** (169/211 lines, up from 78.2%)
+- [x] Commit changes
+- [x] Update this document with actual results
+
+**Note:** Coverage reached 80.1% rather than the estimated 82%. The synonym tests exercised the
+same-position token branch but the remaining 2 uncovered lines (client.rb:297 double-failure rescue,
+index.rb:331 private default body) are in Phase 3 (skipped).
+
+---
+
+## Running Coverage
+
+To generate a coverage report after implementation:
+
+```bash
+# Run specs with coverage
+bundle exec rake coverage
+
+# Or with environment variable
+COVERAGE=true bundle exec rspec
+
+# View report
+open coverage/index.html
+```
+
+To run only the new tests:
+
+```bash
+bundle exec rspec spec/opensearch/sugar/client/settings_spec.rb
+bundle exec rspec spec/opensearch/sugar/index/mappings_spec.rb
+bundle exec rspec spec/opensearch/sugar/client/analyzer_spec.rb
+bundle exec rspec spec/opensearch/sugar/index/analyzer_spec.rb
+```
+
+---
+
+## Notes
+
+- All Phase 1 tests are integration tests requiring a running OpenSearch cluster
+- Use `docker compose up -d` to start the test cluster before running specs
+- The synonym filter tests may need adjustment based on actual OpenSearch synonym behavior
+- Coverage percentages are approximate and may vary slightly based on SimpleCov's line tracking