@prateek_ai/agents-maker 1.0.0

package/config/token_policies.yaml

@@ -0,0 +1,325 @@
+# Token Policies
+# Controls input compression and output verbosity per workflow.
+#
+# Fields per workflow policy:
+#   max_input_files               - max number of file snippets in context
+#   max_input_tokens              - soft limit (tokens); compression triggered above this
+#   history_summarize_after_turns - turn count that triggers history summarization
+#   output_style                  - default output verbosity preset (see output_styles section)
+#   relevance_drop_threshold      - relevance score [0.0–1.0] below which files are excluded
+#   snippet_max_lines             - lines per file before truncation is applied
+#   snippet_head_lines            - lines kept from start of a truncated file
+#   snippet_tail_lines            - lines kept from end of a truncated file
+
+defaults: &defaults
+  max_input_files: 8
+  max_input_tokens: 24000
+  history_summarize_after_turns: 6
+  output_style: standard
+  relevance_drop_threshold: 0.35
+  snippet_max_lines: 200
+  snippet_head_lines: 40
+  snippet_tail_lines: 40
+
+workflows:
+
+  code_review:
+    <<: *defaults
+    max_input_files: 10
+    max_input_tokens: 20000
+    output_style: review_checklist
+    relevance_drop_threshold: 0.40
+
+  feature_implementation:
+    <<: *defaults
+    max_input_files: 6
+    max_input_tokens: 18000
+    history_summarize_after_turns: 5
+    output_style: detailed_with_code
+
+  feature_design:
+    <<: *defaults
+    max_input_files: 5
+    max_input_tokens: 12000
+    output_style: design_brief
+
+  ui_improvement:
+    <<: *defaults
+    max_input_files: 4
+    max_input_tokens: 10000
+    output_style: design_brief
+    snippet_max_lines: 150
+    snippet_head_lines: 60   # keep more of component props + structure
+    snippet_tail_lines: 30
+
+  ux_critique:
+    <<: *defaults
+    max_input_files: 3
+    max_input_tokens: 8000
+    output_style: concise_bullets
+
+  refactoring:
+    <<: *defaults
+    max_input_files: 6
+    max_input_tokens: 16000
+    output_style: detailed_with_code
+    history_summarize_after_turns: 4
+
+  test_generation:
+    <<: *defaults
+    max_input_files: 5
+    max_input_tokens: 14000
+    output_style: detailed_with_code
+
+  generic_project_lifecycle:
+    <<: *defaults
+    max_input_files: 6
+    max_input_tokens: 16000
+    history_summarize_after_turns: 5
+    output_style: standard
+    run_compression_after_phase: true
+
+    phases:
+
+      task_framing:
+        max_input_files: 0
+        max_input_tokens: 4000
+        history_summarize_after_turns: 3
+        output_style: qa_brief
+        run_compression_after_phase: true
+
+      requirements:
+        max_input_files: 5
+        max_input_tokens: 14000
+        history_summarize_after_turns: 4
+        output_style: requirements_spec
+        run_compression_after_phase: true
+
+      solution_design:
+        max_input_files: 5
+        max_input_tokens: 12000
+        history_summarize_after_turns: 4
+        output_style: solution_design
+        run_compression_after_phase: true
+
+      implementation:
+        max_input_files: 6
+        max_input_tokens: 18000
+        history_summarize_after_turns: 5
+        output_style: implementation_slice
+        run_compression_after_phase: false
+
+      review_refinement:
+        max_input_files: 8
+        max_input_tokens: 20000
+        history_summarize_after_turns: 4
+        output_style: critique_summary
+        run_compression_after_phase: true
+
+      handoff:
+        max_input_files: 3
+        max_input_tokens: 8000
+        history_summarize_after_turns: 3
+        output_style: handoff_package
+        run_compression_after_phase: true
+
+    settings:
+      project_state_snapshot_after_phase: true
+      force_state_block_after_turns: 20
+      cross_session_resume_on_project_state: true
+
+    domains:
+
+      software:
+        implementation:
+          max_input_files: 8
+          max_input_tokens: 22000
+        review_refinement:
+          max_input_files: 10
+          output_style: critique_summary
+
+      content:
+        solution_design:
+          max_input_tokens: 8000
+          output_style: solution_design
+        implementation:
+          max_input_files: 3
+          max_input_tokens: 10000
+
+      research:
+        implementation:
+          max_input_files: 4
+          max_input_tokens: 12000
+        review_refinement:
+          max_input_tokens: 16000
+
+      data_analytics:
+        solution_design:
+          max_input_files: 6
+        implementation:
+          max_input_files: 8
+
+      product_design:
+        solution_design:
+          max_input_files: 4
+          max_input_tokens: 10000
+          output_style: solution_design
+        implementation:
+          max_input_files: 3
+          max_input_tokens: 10000
+        review_refinement:
+          max_input_files: 5
+          max_input_tokens: 14000
+
+      marketing:
+        solution_design:
+          max_input_tokens: 8000
+        implementation:
+          max_input_files: 3
+          max_input_tokens: 10000
+
+      ops_process:
+        solution_design:
+          max_input_files: 3
+          max_input_tokens: 8000
+        implementation:
+          max_input_files: 3
+          max_input_tokens: 10000
+        review_refinement:
+          max_input_files: 4
+          max_input_tokens: 12000
+
+      general:
+        task_framing:
+          max_input_tokens: 8000
+          history_summarize_after_turns: 10
+        requirements:
+          max_input_files: 4
+          max_input_tokens: 12000
+        solution_design:
+          max_input_files: 4
+          max_input_tokens: 12000
+        implementation:
+          max_input_files: 5
+          max_input_tokens: 20000
+          history_summarize_after_turns: 8
+        review_refinement:
+          max_input_files: 6
+          max_input_tokens: 18000
+
+# Output verbosity styles
+# These map to the full definitions in token_optimization/output_styles.md.
+# Keys here must match style keys referenced in workflow policies and agent defaults.
+
+output_styles:
+
+  concise_bullets:
+    description: Short bullet lists, no prose paragraphs. Max 300 words per response.
+    max_response_tokens: 600
+    format_hints:
+      - Use bullet lists for all findings.
+      - One sentence per bullet.
+      - No introductory or closing paragraphs.
+      - Omit explanations unless the reason is non-obvious.
+
+  standard:
+    description: Mix of prose and bullets. Balanced detail. Max 600 words per response.
+    max_response_tokens: 1200
+    format_hints:
+      - Use prose for context, bullets for lists of items.
+      - Include brief rationale for recommendations.
+      - One code snippet maximum unless more are essential.
+
+  detailed_with_code:
+    description: >
+      Full explanations with inline code patches or complete file sections.
+      Used for implementation tasks.
+    max_response_tokens: 3000
+    format_hints:
+      - Include full code blocks with language tags.
+      - Explain non-obvious decisions in 1–2 sentences.
+      - Use diff format (+ / -) when showing changes to existing code.
+      - Add a "What changed and why" summary at the end.
+
+  design_brief:
+    description: >
+      Structured design output: tables, bullet lists, short prose.
+      No inline code unless an interface or schema is being defined.
+    max_response_tokens: 1500
+    format_hints:
+      - Use tables for comparisons and API contracts.
+      - Use numbered lists for sequences and steps.
+      - Use bullet lists for options and trade-offs.
+      - Keep each section under 150 words.
+
+  review_checklist:
+    description: >
+      Tabular review output. Each finding has severity, location, and
+      recommendation. No prose paragraphs.
+    max_response_tokens: 1200
+    format_hints:
+      - Output a markdown table with columns: Severity | File:Line | Issue | Recommendation.
+      - Severity values: critical | high | medium | low | info.
+      - One row per finding.
+      - Add a one-line summary count at the top (e.g., "3 high, 2 medium, 1 low").
+
+  # --- Generic Project Lifecycle styles ---
+
+  qa_brief:
+    description: Numbered question list for task framing. Max 5 questions per turn, no prose.
+    max_response_tokens: 500
+    format_hints:
+      - Numbered list of questions only, no preamble.
+      - Each question is one sentence, max 20 words.
+      - After user answers, emit task_profile block and nothing else.
+
+  requirements_spec:
+    description: Structured tables and bullets covering goals, non-goals, deliverables, constraints, assumptions.
+    max_response_tokens: 1200
+    format_hints:
+      - Required sections: Goals, Non-Goals, Target Audience, Deliverables, Constraints, Assumptions.
+      - "Tag each constraint with its type: [technical], [time], [compliance], [resource], [tone]."
+      - No prose paragraphs longer than 2 sentences.
+
+  solution_design:
+    description: Structured design artifact with Context, Approach, Structure, Risks sections.
+    max_response_tokens: 2000
+    format_hints:
+      - Required sections: Context, Approach, Structure, Risks & Open Questions.
+      - Structure section varies by domain (see docs/domains.md).
+      - Tables for comparisons; numbered lists for sequences.
+      - No implementation code; schemas and interfaces acceptable.
+      - End with numbered Open Questions list.
+
+  implementation_slice:
+    description: Increment plan + work content + approval prompt. One increment at a time.
+    max_response_tokens: 2500
+    format_hints:
+      - Start with 3-line Increment Plan (this slice, depends on, next slice).
+      - Software uses diff format; content uses section heading + body + transition note.
+      - End with "Approve this increment / request changes / change direction?"
+
+  critique_summary:
+    description: Verdict + severity-rated findings table + positive findings + action prompt.
+    max_response_tokens: 1500
+    format_hints:
+      - Open with one-line verdict (ready_to_ship | minor_revisions | significant_revisions).
+      - Table columns: Severity | Area | Issue | Recommendation.
+      - Severity: critical, high, medium, low, info.
+      - Minimum 2 positive findings.
+      - End with "Apply all fixes / apply selected fixes / discuss?"
+
+  handoff_package:
+    description: Summary bullets + how-to steps + prioritized next steps. No prose paragraphs.
+    max_response_tokens: 1200
+    format_hints:
+      - Required sections: Summary (3-5 bullets), How to Use/Run/Deploy, What's Done, What's Next.
+      - What's Next must use P1/P2/P3 priority tiers.
+      - Domain-specific additions (env vars for software, key findings table for research, etc.).
+
+# Compression triggers
+# The Compression Agent is invoked automatically when:
+compression_triggers:
+  context_token_ratio: 0.75   # context exceeds 75% of model's max input tokens
+  history_turns_exceeded: true # turns exceed history_summarize_after_turns for the workflow
+  explicit_user_request: true  # user says "summarize", "compress", "reduce context"

package/context_loaders/__init__.py

@@ -0,0 +1,15 @@
+"""
+context_loaders
+
+Utilities for generating compact project context for agent sessions.
+
+Scripts:
+    repo_tree.py       - Walk repo and emit a filtered file tree with descriptions.
+    file_chunker.py    - Extract and truncate key files for agent context.
+    project_summary.py - Produce a compact stack/services/entrypoints summary.
+
+Typical session setup:
+    python context_loaders/project_summary.py --path /your/repo
+    python context_loaders/repo_tree.py --path /your/repo --filter src/ app/
+    (paste both outputs as the opening message in your agent session)
+"""

package/context_loaders/file_chunker.py

@@ -0,0 +1,252 @@
+"""
+context_loaders/file_chunker.py
+
+Extract, read, and optionally truncate specific files for inclusion in an
+agent context block. Designed to be used after repo_tree.py identifies
+the relevant files.
+
+Usage:
+    # Read specific files
+    python context_loaders/file_chunker.py \\
+        --path /your/repo \\
+        --files services/user_service.py models/user.py tests/test_user_service.py
+
+    # Read all files under a directory, truncated
+    python context_loaders/file_chunker.py \\
+        --path /your/repo \\
+        --dirs services/ models/ \\
+        --max-lines 200
+
+    # Output to a file
+    python context_loaders/file_chunker.py \\
+        --path /your/repo \\
+        --files services/user_service.py \\
+        --output chunks.txt
+
+Output format (paste directly into agent context):
+    ### services/user_service.py
+    ```python
+    <file content or truncated snippet>
+    ```
+"""
+
+from __future__ import annotations
+
+__version__ = "1.0.0"
+
+import argparse
+import sys
+from pathlib import Path
+
+# Default truncation settings (matches token_optimization defaults)
+DEFAULT_MAX_LINES = 200
+DEFAULT_HEAD_LINES = 40
+DEFAULT_TAIL_LINES = 40
+
+# Extensions that have a known language tag for fenced code blocks
+LANG_MAP: dict[str, str] = {
+    ".py": "python",
+    ".ts": "typescript",
+    ".tsx": "tsx",
+    ".js": "javascript",
+    ".jsx": "jsx",
+    ".go": "go",
+    ".java": "java",
+    ".rb": "ruby",
+    ".rs": "rust",
+    ".cs": "csharp",
+    ".cpp": "cpp",
+    ".c": "c",
+    ".sql": "sql",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".toml": "toml",
+    ".json": "json",
+    ".md": "markdown",
+    ".sh": "bash",
+    ".bash": "bash",
+    ".graphql": "graphql",
+    ".gql": "graphql",
+    ".proto": "protobuf",
+    ".html": "html",
+    ".css": "css",
+    ".scss": "scss",
+}
+
+
+def _lang_tag(path: Path) -> str:
+    return LANG_MAP.get(path.suffix.lower(), "text")
+
+
+def truncate_content(content: str, max_lines: int, head_lines: int, tail_lines: int) -> tuple[str, int]:
+    """
+    Truncate content to head + tail with a gap marker.
+    Returns (truncated_content, lines_omitted).
+    """
+    lines = content.splitlines()
+    if len(lines) <= max_lines:
+        return content, 0
+
+    head = lines[:head_lines]
+    tail = lines[-tail_lines:]
+    omitted = len(lines) - head_lines - tail_lines
+    gap = f"# ... [{omitted} lines omitted — request full file if needed] ..."
+    return "\n".join(head + [gap] + tail), omitted
+
+
+def read_file(
+    file_path: Path,
+    max_lines: int = DEFAULT_MAX_LINES,
+    head_lines: int = DEFAULT_HEAD_LINES,
+    tail_lines: int = DEFAULT_TAIL_LINES,
+    no_truncate: bool = False,
+) -> dict:
+    """
+    Read a single file and return a dict with path, content, lang, and truncation info.
+    """
+    try:
+        content = file_path.read_text(encoding="utf-8", errors="replace")
+    except (OSError, PermissionError) as e:
+        return {
+            "path": str(file_path),
+            "content": f"# Error reading file: {e}",
+            "lang": "text",
+            "truncated": False,
+            "lines_omitted": 0,
+            "total_lines": 0,
+        }
+
+    total_lines = content.count("\n") + 1
+
+    if no_truncate:
+        truncated_content, omitted = content, 0
+    else:
+        truncated_content, omitted = truncate_content(content, max_lines, head_lines, tail_lines)
+
+    return {
+        "path": str(file_path),
+        "content": truncated_content,
+        "lang": _lang_tag(file_path),
+        "truncated": omitted > 0,
+        "lines_omitted": omitted,
+        "total_lines": total_lines,
+    }
+
+
+def format_chunk(chunk: dict, root: Path | None = None) -> str:
+    """Format a single file chunk as a fenced code block for agent context."""
+    display_path = chunk["path"]
+    if root:
+        try:
+            display_path = str(Path(chunk["path"]).relative_to(root))
+        except ValueError:
+            pass
+
+    header = f"### {display_path}"
+    if chunk["truncated"]:
+        header += f"  _(truncated: {chunk['lines_omitted']} lines omitted of {chunk['total_lines']} total)_"
+
+    return f"{header}\n```{chunk['lang']}\n{chunk['content']}\n```"
+
+
+def collect_from_dirs(
+    root: Path,
+    dirs: list[str],
+    extensions: set[str] | None = None,
+    max_files: int = 20,
+) -> list[Path]:
+    """Collect file paths from specified subdirectories."""
+    if extensions is None:
+        extensions = set(LANG_MAP.keys())
+
+    paths: list[Path] = []
+    for d in dirs:
+        target = root / d.lstrip("/")
+        if not target.is_dir():
+            print(f"Warning: {target} is not a directory — skipping.", file=sys.stderr)
+            continue
+        for p in sorted(target.rglob("*")):
+            if p.is_file() and p.suffix in extensions:
+                paths.append(p)
+            if len(paths) >= max_files:
+                break
+    return paths[:max_files]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Extract and format file chunks for agent context."
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    parser.add_argument("--path", required=True, help="Repository root directory.")
+    parser.add_argument(
+        "--files", nargs="*", metavar="FILE",
+        help="Relative paths to specific files (from --path root)."
+    )
+    parser.add_argument(
+        "--dirs", nargs="*", metavar="DIR",
+        help="Subdirectories to collect all key files from."
+    )
+    parser.add_argument("--max-lines", type=int, default=DEFAULT_MAX_LINES,
+                        help=f"Max lines before truncation (default: {DEFAULT_MAX_LINES}).")
+    parser.add_argument("--head-lines", type=int, default=DEFAULT_HEAD_LINES,
+                        help=f"Lines kept from start on truncation (default: {DEFAULT_HEAD_LINES}).")
+    parser.add_argument("--tail-lines", type=int, default=DEFAULT_TAIL_LINES,
+                        help=f"Lines kept from end on truncation (default: {DEFAULT_TAIL_LINES}).")
+    parser.add_argument("--no-truncate", action="store_true",
+                        help="Disable truncation (caution: may produce very large output).")
+    parser.add_argument("--max-files", type=int, default=20,
+                        help="Max files when using --dirs (default: 20).")
+    parser.add_argument("--output", help="Write output to this file instead of stdout.")
+    args = parser.parse_args()
+
+    root = Path(args.path).resolve()
+    if not root.is_dir():
+        print(f"Error: {root} is not a directory.", file=sys.stderr)
+        sys.exit(1)
+
+    file_paths: list[Path] = []
+
+    if args.files:
+        root_resolved = root.resolve()
+        for f in args.files:
+            p = (root / f).resolve()
+            if not str(p).startswith(str(root_resolved)):
+                print(f"[WARN] Skipping {f} — path escapes project root.", file=sys.stderr)
+                continue
+            if p.is_file():
+                file_paths.append(p)
+            else:
+                print(f"[WARN] {p} not found — skipping.", file=sys.stderr)
+
+    if args.dirs:
+        file_paths.extend(collect_from_dirs(root, args.dirs, max_files=args.max_files))
+
+    if not file_paths:
+        print("No files found. Provide --files or --dirs.", file=sys.stderr)
+        sys.exit(1)
+
+    chunks = [
+        read_file(
+            p,
+            max_lines=args.max_lines,
+            head_lines=args.head_lines,
+            tail_lines=args.tail_lines,
+            no_truncate=args.no_truncate,
+        )
+        for p in file_paths
+    ]
+
+    header = f"## Relevant Files ({len(chunks)} files)\n"
+    body = "\n\n".join(format_chunk(c, root=root) for c in chunks)
+    output = header + "\n" + body + "\n"
+
+    if args.output:
+        Path(args.output).write_text(output, encoding="utf-8")
+        print(f"Written to {args.output}")
+    else:
+        sys.stdout.buffer.write(output.encode("utf-8"))
+
+
+if __name__ == "__main__":
+    main()