coupler 0.0.1-java

data/lib/coupler/models/transformer.rb

@@ -0,0 +1,110 @@
+module Coupler
+  module Models
+    class Transformer < Sequel::Model
+      include CommonModel
+
+      plugin :serialization, :marshal, :allowed_types
+
+      TYPES = %w{string integer datetime}
+      EXAMPLES = {
+        'string'   => 'Test',
+        'integer'  => 123,
+        'datetime' => lambda { Time.now }
+      }
+
+      def accepts_type?(type)
+        allowed_types.is_a?(Array) && allowed_types.include?(type)
+      end
+
+      def transform(data, options)
+        input = data[options[:in]]
+        runner = Runner.new(code, input)
+        output = runner.run
+        data[options[:out]] = output
+        data
+      end
+
+      def preview
+        return nil  if allowed_types.nil? || code.nil? || code == ""
+
+        result = {'success' => true}
+        examples = EXAMPLES.reject { |k, v| !allowed_types.include?(k) }
+        examples.each_pair do |type, obj|
+          obj = obj.call  if obj.is_a?(Proc)
+          result[type] = { :in => obj }
+          begin
+            transform(result[type], {:in => :in, :out => :out})
+
+            expected_type = result_type == 'same' ? type : result_type
+            actual_type = case result[type][:out]
+                          when String then "string"
+                          when Fixnum then "integer"
+                          when Time, Date, DateTime then "datetime"
+                          when NilClass then "null"
+                          end
+
+            if actual_type != "null" && expected_type != actual_type
+              raise TypeError, "expected #{expected_type}, got #{actual_type}"
+            end
+
+          rescue Exception => e
+            result[type][:out] = e
+            result['success'] = false
+          end
+        end
+        result
+      end
+
+      def field_changes(*fields)
+        fields.inject({}) do |result, field|
+          result[field.id] = hash = {}
+          if result_type != 'same'
+            hash[:type] = result_type.to_sym
+
+            # FIXME: don't use db_type anymore
+            hash[:db_type] =
+              case result_type
+              when 'integer'  then 'int(11)'
+              when 'string'   then 'varchar(255)'
+              when 'datetime' then 'datetime'
+              end
+          end
+          result
+        end
+      end
+
+      private
+        def validate
+          super
+          validates_presence [:name, :allowed_types, :result_type, :code]
+          validates_unique :name
+          validates_includes TYPES + ['same'], :result_type
+
+          if errors.on(:allowed_types).nil?
+            bad = (allowed_types - TYPES).uniq
+            if !bad.empty?
+              errors.add(:allowed_types, "has invalid type(s): #{bad.join(', ')}")
+            end
+          end
+
+          if errors.on(:code).nil?
+            io = java.io.ByteArrayInputStream.new(code.to_java_bytes)
+            begin
+              JRuby.runtime.parseInline(io, "line", nil)
+            rescue Exception => e
+              errors.add(:code, "has errors: #{e.to_s}")
+            end
+          end
+
+          if errors.empty?
+            result = preview
+            if !(result && result['success'])
+              errors.add(:code, "has errors")
+            end
+          end
+        end
+    end
+  end
+end
+
+require File.join(File.dirname(__FILE__), "transformer", "runner")

data/lib/coupler/models.rb

@@ -0,0 +1,30 @@
+module Coupler
+  module Models
+    # NOTE: using autoload here would undoubtedly be more efficient, but
+    # I need to make sure the database connection is instantiated before
+    # loading these classes because of how Sequel::Model works.
+    #%w{connection project resource field transformer transformation scenario matcher job result comparison}.each do |name|
+      #autoload(name.capitalize.to_sym, File.dirname(__FILE__) + "/models/#{name}")
+    #end
+
+    NAMES = [
+      :Connection, :Project, :Resource, :Field, :Transformer,
+      :Transformation, :Scenario, :Matcher, :Job, :Result, :Comparison,
+      :Import
+    ]
+    def self.const_missing(name)
+      name = name.to_sym
+      if NAMES.include?(name)
+        Database.instance
+        require File.dirname(__FILE__) + "/models/#{name.to_s.downcase}"
+        const_get(name)
+      else
+        puts "#{name.inspect} wasn't in #{NAMES.inspect}"
+        super
+      end
+    end
+  end
+end
+
+require File.dirname(__FILE__) + "/models/common_model"
+require File.dirname(__FILE__) + "/models/jobify"

data/lib/coupler/runner.rb

@@ -0,0 +1,76 @@
+module Coupler
+  class Runner
+    def initialize(argv = ARGV)
+      irb = false
+      OptionParser.new do |opts|
+        opts.on("-p", "--port PORT", "Web server port") do |port|
+          Base.set(:port, port.to_i)
+        end
+        opts.on("--dir DIR", "Directory to use for Coupler's data") do |dir|
+          Base.set(:data_path, dir)
+        end
+        opts.on("-e", "--environment ENVIRONMENT", "Set the environment") do |env|
+          case env
+          when "production", "development", "test"
+            Base.set(:environment, env.to_sym)
+          else
+            raise "Invalid environment (must be production, development, or test)"
+          end
+        end
+        opts.on('-i', '--interactive', "Run an IRB session") do
+          irb = true
+        end
+      end.parse!(argv)
+
+      puts "Migrating database..."
+      Coupler::Database.instance.migrate!
+
+      puts "Starting scheduler..."
+      Coupler::Scheduler.instance.start
+
+      puts "Starting web server..."
+      handler = Rack::Handler.get('mongrel')
+      settings = Coupler::Base.settings
+
+      # See the Rack::Handler::Mongrel.run! method
+      # NOTE: I don't want to join the server immediately, which is why I'm
+      #       doing this by hand.
+      @web_server = Mongrel::HttpServer.new(settings.bind, settings.port, 950, 0, 60)
+      @web_server.register('/', handler.new(Coupler::Base))
+      success = false
+      begin
+        @web_thread = @web_server.run
+        success = true
+      rescue Errno::EADDRINUSE => e
+        Scheduler.instance.shutdown
+        puts "Can't start web server, port already in use. Aborting..."
+      end
+
+      if success
+        Coupler::Base.set(:running, true)
+        trap("INT") do
+          shutdown
+        end
+
+        puts <<'EOF'
+                             ___
+                            /\_ \
+  ___    ___   __  __  _____\//\ \      __   _ __
+ /'___\ / __`\/\ \/\ \/\ '__`\\ \ \   /'__`\/\`'__\
+/\ \__//\ \L\ \ \ \_\ \ \ \L\ \\_\ \_/\  __/\ \ \/
+\ \____\ \____/\ \____/\ \ ,__//\____\ \____\\ \_\
+ \/____/\/___/  \/___/  \ \ \/ \/____/\/____/ \/_/
+                         \ \_\
+                          \/_/
+EOF
+        @web_thread.join
+      end
+    end
+
+    def shutdown
+      puts "Shutting down..."
+      Scheduler.instance.shutdown
+      @web_server.stop
+    end
+  end
+end

data/lib/coupler/scheduler.rb

@@ -0,0 +1,56 @@
+module Coupler
+  class Scheduler
+    include Singleton
+
+    def initialize
+      super
+      @mutex = Mutex.new
+    end
+
+    def schedule_transform_job(resource)
+      Models::Job.create({
+        :name => "transform",
+        :resource => resource,
+        :status => "scheduled"
+      })
+    end
+
+    def schedule_run_scenario_job(scenario)
+      Models::Job.create({
+        :name => "run_scenario",
+        :scenario => scenario,
+        :status => "scheduled"
+      })
+    end
+
+    def run_jobs
+      @mutex.synchronize do
+        count = Models::Job.filter(:status => 'running').count
+        if count == 0
+          job = Models::Job.filter(:status => 'scheduled').order(:created_at).first
+          Thread.new(job) { |j| j.execute } if job
+        end
+      end
+    end
+
+    def start
+      if !is_started?
+        @loop = Thread.new do
+          loop do
+            sleep 30
+            run_jobs
+          end
+        end
+      end
+    end
+
+    def shutdown
+      @loop.exit
+      @loop = nil
+    end
+
+    def is_started?
+      !@loop.nil?
+    end
+  end
+end

data/lib/coupler.rb

@@ -0,0 +1,34 @@
+require 'java'
+require 'jruby/core_ext'
+
+require 'fileutils'
+require 'erb'
+require 'delegate'
+require 'singleton'
+require 'logger'
+require 'optparse'
+require 'thwait'
+require 'digest'
+require 'rack'
+require 'rack/mime'   # This is an attempt to avoid NameError exceptions
+require 'sinatra/base'
+require 'rack/flash'
+require 'sequel'
+require 'sequel/extensions/migration'
+require 'json'
+require 'fastercsv'
+require 'carrierwave'
+require 'mvn:com.h2database:h2'
+require 'mongrel'
+require 'jdbc/mysql'  # FIXME: lazy load this
+
+require File.dirname(__FILE__) + "/coupler/logger"
+require File.dirname(__FILE__) + "/coupler/database"
+require File.dirname(__FILE__) + "/coupler/scheduler"
+require File.dirname(__FILE__) + "/coupler/data_uploader"
+require File.dirname(__FILE__) + "/coupler/import_buffer"
+require File.dirname(__FILE__) + "/coupler/models"
+require File.dirname(__FILE__) + "/coupler/extensions"
+require File.dirname(__FILE__) + "/coupler/helpers"
+require File.dirname(__FILE__) + "/coupler/runner"
+require File.dirname(__FILE__) + "/coupler/base"

data/log/.gitignore

@@ -0,0 +1 @@
+*.log

data/misc/README

@@ -0,0 +1,5 @@
+There was no license distributed with rack-flash.  I found it here:
+  http://nakajima.github.com/rack-flash/
+
+Same thing for json-jruby.  Found it here:
+  http://github.com/mernen/json-jruby/raw/master/COPYING

data/misc/jruby-json.license

@@ -0,0 +1,57 @@
+JSON-JRuby is copyrighted free software by Daniel Luz <mernen at gmail dot com>,
+and is a derivative work of Florian Frank's json library <flori at ping dot de>.
+You can redistribute it and/or modify it under either the terms of the GPL
+version 2 (see the file GPL), or the conditions below:
+
+  1. You may make and give away verbatim copies of the source form of the
+     software without restriction, provided that you duplicate all of the
+     original copyright notices and associated disclaimers.
+
+  2. You may modify your copy of the software in any way, provided that
+     you do at least ONE of the following:
+
+       a) place your modifications in the Public Domain or otherwise
+          make them Freely Available, such as by posting said
+	  modifications to Usenet or an equivalent medium, or by allowing
+	  the author to include your modifications in the software.
+
+       b) use the modified software only within your corporation or
+          organization.
+
+       c) give non-standard binaries non-standard names, with
+          instructions on where to get the original software distribution.
+
+       d) make other distribution arrangements with the author.
+
+  3. You may distribute the software in object code or binary form,
+     provided that you do at least ONE of the following:
+
+       a) distribute the binaries and library files of the software,
+	  together with instructions (in the manual page or equivalent)
+	  on where to get the original distribution.
+
+       b) accompany the distribution with the machine-readable source of
+	  the software.
+
+       c) give non-standard binaries non-standard names, with
+          instructions on where to get the original software distribution.
+
+       d) make other distribution arrangements with the author.
+
+  4. You may modify and include the part of the software into any other
+     software (possibly commercial).  But some files in the distribution
+     are not written by the author, so that they are not under these terms.
+
+     For the list of those files and their copying conditions, see the
+     file LEGAL.
+
+  5. The scripts and library files supplied as input to or produced as 
+     output from the software do not automatically fall under the
+     copyright of the software, but belong to whomever generated them, 
+     and may be sold commercially, and may be aggregated with this
+     software.
+
+  6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+     IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+     WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+     PURPOSE.

data/misc/rack-flash.license

@@ -0,0 +1,22 @@
+Copyright (c) 2009 Pat Nakajima
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/script/dbconsole.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'lib/coupler'
+
+server = Coupler::Server.instance
+server.console || puts("Coupler's database server isn't running.  Start it with `rake db:console'")

data/src/edu/vanderbilt/coupler/Main.java

@@ -0,0 +1,116 @@
+package edu.vanderbilt.coupler;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import java.util.ArrayList;
+import java.util.List;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.jruby.embed.ScriptingContainer;
+import org.jruby.Ruby;
+import org.jruby.RubyArray;
+import org.jruby.RubyString;
+
+public class Main {
+  private static final Properties couplerProperties = new Properties();
+  public static final String COUPLER_PROPERTIES = "coupler.properties";
+  public static final String JRUBY_PROPERTIES = "jruby.properties";
+
+  static {
+    InputStream stream = null;
+    try {
+      stream = Main.class.getResourceAsStream(COUPLER_PROPERTIES);
+      if (stream == null) {
+        throw new RuntimeException("Resource not found: " + COUPLER_PROPERTIES);
+      }
+      couplerProperties.load(stream);
+    } catch (IOException ioe) {
+      ioe.printStackTrace();
+    } finally {
+      if (stream != null) {
+        try {
+          stream.close();
+        } catch (IOException e) {
+          // silently ignore
+        }
+      }
+    }
+  }
+
+  private Main(String[] args) {
+    System.out.printf("Coupler version: %s\nBuild date: %s\n\n",
+        couplerProperties.getProperty("coupler.version"),
+        couplerProperties.getProperty("build.timestamp"));
+
+    // Set JRuby runtime properties
+    Properties systemProperties;
+    InputStream stream = null;
+    try {
+      stream = Main.class.getResourceAsStream(JRUBY_PROPERTIES);
+      if (stream == null) {
+        throw new RuntimeException("Resource not found: " + JRUBY_PROPERTIES);
+      }
+      systemProperties = new Properties(System.getProperties());
+      systemProperties.load(stream);
+      System.setProperties(systemProperties);
+    } catch (IOException ioe) {
+      ioe.printStackTrace();
+    } finally {
+      if (stream != null) {
+        try {
+          stream.close();
+        } catch (IOException e) {
+          // silently ignore
+        }
+      }
+    }
+
+    String location = findCouplerPath();
+    List<String> loadPaths = new ArrayList();
+    loadPaths.add(location);
+
+    ScriptingContainer container = new ScriptingContainer();
+    container.setLoadPaths(loadPaths);
+
+    // FIXME: I feel like there's a simpler way of doing this.
+    Ruby ruby = container.getProvider().getRuntime();
+    RubyArray rbArray = ruby.newArray();
+    for (String string : args) {
+      rbArray.append(ruby.newString(string));
+    }
+    container.put("argv", rbArray);
+
+    // SystemExit gets thrown when someone runs the JAR with --help
+    String script =
+      "require 'coupler'\n" +
+      "begin\n" +
+      "  Coupler::Runner.new(argv)\n" +
+      "rescue SystemExit\n" +
+      "end";
+    container.runScriptlet(script);
+  }
+
+  private String findCouplerPath() {
+    try {
+      URL resource = getClass().getResource("/META-INF/coupler.home/lib/coupler.rb");
+      String location = resource.toURI().getSchemeSpecificPart();
+      Pattern p = Pattern.compile("coupler\\.rb$");
+      Matcher m = p.matcher(location);
+      while(m.find()) {
+        location = location.substring(0, m.start() - 1);
+        return location;
+      }
+      return null;
+    }
+    catch (URISyntaxException e) {
+      return null;
+    }
+  }
+
+  public static void main(String[] args) {
+    new Main(args);
+  }
+}

data/src/edu/vanderbilt/coupler/jruby.properties

@@ -0,0 +1 @@
+jruby.thread.pooling=true

data/tasks/annotations.rake

@@ -0,0 +1,84 @@
+class SourceAnnotationExtractor
+  class Annotation < Struct.new(:line, :tag, :text)
+    def to_s(options={})
+      s = "[%3d] " % line
+      s << "[#{tag}] " if options[:tag]
+      s << text
+    end
+  end
+
+  def self.enumerate(tag, options={})
+    extractor = new(tag)
+    extractor.display(extractor.find, options)
+  end
+
+  attr_reader :tag
+
+  def initialize(tag)
+    @tag = tag
+  end
+
+  def find(dirs=%w(lib test))
+    dirs.inject({}) { |h, dir| h.update(find_in(dir)) }
+  end
+
+  def find_in(dir)
+    results = {}
+
+    Dir.glob("#{dir}/*") do |item|
+      next if File.basename(item)[0] == ?.
+
+      if File.directory?(item)
+        results.update(find_in(item))
+      elsif item =~ /\.(builder|(r(?:b|xml|js)))$/
+        results.update(extract_annotations_from(item, /#\s*(#{tag}):?\s*(.*)$/))
+      elsif item =~ /\.(rhtml|erb)$/
+        results.update(extract_annotations_from(item, /<%\s*#\s*(#{tag}):?\s*(.*?)\s*%>/))
+      end
+    end
+
+    results
+  end
+
+  def extract_annotations_from(file, pattern)
+    lineno = 0
+    result = File.readlines(file).inject([]) do |list, line|
+      lineno += 1
+      next list unless line =~ pattern
+      list << Annotation.new(lineno, $1, $2)
+    end
+    result.empty? ? {} : { file => result }
+  end
+
+  def display(results, options={})
+    results.keys.sort.each do |file|
+      puts "#{file}:"
+      results[file].each do |note|
+        puts "  * #{note.to_s(options)}"
+      end
+      puts
+    end
+  end
+end
+
+desc "Enumerate all annotations"
+task :notes do
+  SourceAnnotationExtractor.enumerate "OPTIMIZE|FIXME|TODO", :tag => true
+end
+
+namespace :notes do
+  desc "Enumerate all OPTIMIZE annotations"
+  task :optimize do
+    SourceAnnotationExtractor.enumerate "OPTIMIZE"
+  end
+
+  desc "Enumerate all FIXME annotations"
+  task :fixme do
+    SourceAnnotationExtractor.enumerate "FIXME"
+  end
+
+  desc "Enumerate all TODO annotations"
+  task :todo do
+    SourceAnnotationExtractor.enumerate "TODO"
+  end
+end