talia_core 0.4.1 → 0.4.2

data/lib/talia_util/rake_tasks.rb

@@ -0,0 +1,223 @@
+# Rake tasks for the talia core
+
+require 'rake'
+require 'yaml'
+require 'talia_core'
+require 'talia_util'
+require 'talia_util/util'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'progressbar'
+
+include TaliaUtil
+
+namespace :talia_core do
+  
+  # Standard initialization
+  desc "Initialize the TaliaCore"
+  task :talia_init do
+    Util::title
+    Util::init_talia
+    TLoad::force_rails_parts
+    Util::talia_config if(Util::flag?('verbose'))
+    # Add load paths to allow autoloading of all class from this tasks
+    ActiveSupport::Dependencies.load_paths << File.join(TALIA_ROOT, 'lib')
+    model_path = File.join(TALIA_ROOT, 'app', 'models')
+    ActiveSupport::Dependencies.load_paths << model_path if(File.directory?(model_path))
+  end
+  
+  # Removes all data
+  desc "Reset the Talia data store"
+  task :clear_store => :talia_init do
+    Util::flush_db
+    Util::flush_rdf
+    puts "Flushed data store"
+  end
+  
+  # Init for the unit tests
+  desc "Initialize Talia for the tests"
+  task :test_setup do
+    unless(ENV['environment'])
+      puts "Setting environment to 'test'"
+      ENV['environment'] = "test"
+    end
+
+    # Invoke the init after the setup
+    Rake::Task["talia_core:talia_init"].invoke
+  end
+  
+  # Test task
+  desc 'Test the talia_core plugin.'
+  task :test => :test_setup
+  # Create the test tasks
+  Rake::TestTask.new(:test) do |t| 
+    t.libs << 'lib'
+    # This will always take the files from the talia_core directory
+    t.test_files = FileList["#{File.dirname(__FILE__)}/../test/**/*_test.rb"]
+    t.verbose = true
+  end
+  
+  # Queue an import in the background
+  desc "Xml Background import. Options: [index=<indexfile>] [xml=<datafile>] [importer=<importclass>] [reset_store=true] [...]"
+  task :xml_background_import do 
+    background_job('xml_import', :tag => 'import')
+    puts "Queued XML background import."
+  end
+  
+  desc "Xml Import. Options: see xml_background_import"
+  task :xml_import => :talia_init do
+    importer = TaliaUtil::ImportJobHelper.new(STDOUT, TaliaUtil::BarProgressor)
+    importer.do_import
+  end
+  
+  # Just run the Talia init to test it
+  desc "Test the TaliaCore startup"
+  task :init_test => :talia_init do
+    Util::talia_config
+  end
+ 
+  # Task for importing ontologies/raw RDF data
+  desc "Import ontologies. This imports the given rdf files (same as rdf_import), and sets the context automatically"
+  task :ontology_import => :talia_init do
+    TaliaCore::RdfImport::import(ENV['rdf_syntax'], TaliaUtil::Util::get_files, :auto)
+  end
+  
+  # RDF importing task. A context can be freely assigned.
+  desc "Import RDF data directly into the triple store. Option: rdf_syntax={ntriples|rdfxml} [context=<context>]"
+  task :rdf_import => :talia_init do
+    TaliaCore::RdfImport::import(ENV['rdf_syntax'], TaliaUtil::Util::get_files, ENV['context'])
+  end
+
+  desc "Update the Ontologies. Options [ontologies=<ontology_folder>]"
+  task :setup_ontologies => :talia_init do
+    Util::setup_ontologies
+  end
+
+  # Rewrite your base URL. This will loose any comments in the config file
+  desc "Rewrite the database to move it to a new URL. Options new_home=<url>."
+  task :move_site => :talia_init do
+    new_site = ENV['new_home']
+    # Check if this looks like an URL
+    raise(RuntimeError, "Illegal new_home given. (It must start with http(s):// and end with a slash)") unless(new_site =~ /^https?:\/\/\S+\/$/)
+    # open up the configuration file
+    config_file_path = File.join(TALIA_ROOT, 'config', 'talia_core.yml')
+    config = YAML::load(File.open(config_file_path))
+    old_site = config['local_uri']
+    raise(RuntimeError, "Could not determine current local URI") unless(old_site && old_site.strip != '')
+    puts "New home URL: #{new_site}"
+    puts "Original home URL: #{old_site}"
+    # Rewrite the sql database
+    ActiveRecord::Base.connection.execute("UPDATE active_sources SET uri = replace(uri, '#{old_site}', '#{new_site}')")
+    puts('Updated database, now recreating RDF')
+    # Rebuild the RDF
+    prog = ProgressBar.new('Rebuilding', Util::rewrite_count)
+    Util::rewrite_rdf { prog.inc }
+    prog.finish
+    # Rebuild the ontologies
+    Util::setup_ontologies
+    # Write back to the config file
+    config['local_uri'] = new_site
+    open(config_file_path, 'w') { |io| io.puts(config.to_yaml) }
+    puts "New configuration saved. Finished site rebuilding."
+  end
+
+  # Task for importing YAML data into the data store
+  desc "Import YAML data file in Talia format."
+  task :yaml_import => :talia_init do
+    YamlImport::import_multi_files(TaliaUtil::Util::get_files)
+  end
+  
+  # Task for updating the OWL classes with RDFS class information
+  desc "Update OWL classes with RDFS class information."
+  task :owl_to_rdfs_update => :talia_init do
+    RdfUpdate::owl_to_rdfs
+  end
+  
+  # Task to import data files into the Talia system
+  desc "Import data files. Options data_type=<data_type> replace_files={yes|no}"
+  task :data_import => :talia_init do
+    DataImport::import(TaliaUtil::Util::get_files, ENV['data_type'])
+  end
+  
+  # Task to import demo data from a demo directory
+  desc "Import demo data (for default demo data). Opions: demodir=<dir> [owlify=no]"
+  task :demo_import do
+    unless(demodir = ENV['demodir'])
+      puts "ERROR: Need demodir option for import"
+      Util::print_options
+      exit(1)
+    end
+    
+    # Force some options to default
+    ENV['reset_db'] = "yes" unless(ENV['reset_db'])
+    ENV['reset_rdf'] = "yes" unless(ENV['reset_rdf'])
+    
+    # Invoke the init after the setup
+    Rake::Task["talia_core:talia_init"].invoke
+    
+    puts "Importing ontologies..."
+    RdfImport::import("rdfxml", FileList.new(File.join(demodir, '*.rdf*'), File.join(demodir, '*.owl')))
+    puts "Importing data records..."
+    YamlImport::import_multi_files([File.join(demodir, "demo_data.yml")])
+    puts "Importing files..."
+    Dir.foreach(File.join(demodir)) do |entry|
+      if(FileTest.directory?(File.join(demodir,entry)) && entry != ".." && entry != "." && entry != ".svn")
+        puts "Importing for type #{entry}"
+        DataImport::import(FileList.new(File.join(demodir, entry, '*')), entry)
+      end
+    end
+    if(ENV['owlify'].to_s.downcase == 'yes')
+      RdfUpdate::owl_to_rdfs
+    end
+  end
+  
+  # Helper task to bootstrap Redland RDF (should usually only be a problem when
+  # using Redland with mysql store)
+  desc "Initialize Redland RDF store. Option: rdfconf=<rdfconfig_file> [environment=env]"
+  task :redland_init do
+    # This simply activates the RDF store once with the :new option set.
+    Util.title
+    environment = ENV['environment'] || "development"
+    raise(ArgumentError, "Must have rdfconf=<config_file>") unless(ENV['rdfconf'])
+    options = YAML::load(File.open(ENV['rdfconf']))[environment]
+    
+    rdf_cfg = Hash.new
+    options.each { |key, value| rdf_cfg[key.to_sym] = value }
+    
+    rdf_cfg[:new] = "yes"
+    
+    ConnectionPool.add_data_source(rdf_cfg)
+  end
+
+  # Help info
+  desc "Help on general options for the TaliaCore tasks"
+  task :help do
+    Util.title
+    puts "Talia Core tasks usage information."
+    Util::print_options
+  end
+
+  desc "Rebuild the RDF store from the database. Option [hard_reset=(true|false)]"
+  task :rebuild_rdf => :talia_init do
+    count = TaliaCore::SemanticRelation.count
+    puts "Rebuilding RDF for #{count} triples."
+    prog = ProgressBar.new('Rebuilding', count)
+    Util::rewrite_rdf { prog.inc }
+    prog.finish
+    puts "Finished rewriting. ATTENTION: You may want to call setup_ontologies now."
+  end
+  
+  # Helper methods
+  
+  # Queue the long-running task in the background processing queue.
+  # This will simply queue the job, and doesn't start the runner
+  # by itself
+  def background_job(job, options)
+    # Use the current environment for the job
+    options[:env] = ENV.merge(options[:env] || {})
+    # Avoid "tickling" to start the background job from here
+    options[:no_tickle] = true
+    TaliaCore::BackgroundJobs::Job.submit_with_progress(job, options)
+  end
+  
+end

data/tasks/talia_core_tasks.rake

@@ -0,0 +1,2 @@
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'talia_util/rake_tasks'

metadata

@@ -5,10 +5,143 @@ homepage: http://trac.talia.discovery-project.eu/
 executables: 
 - talia
 version: !ruby/object:Gem::Version 
-  version: 0.4.1
+  version: 0.4.2
 post_install_message: 
-date: 2009-10-13 22:00:00 +00:00
+date: 2009-10-15 22:00:00 +00:00
 files: 
+- VERSION.yml
+- config/database.yml
+- config/database.yml.example
+- config/rdfstore.yml
+- config/rdfstore.yml.example
+- config/rdfstore.yml.rdflite_example
+- config/rdfstore.yml.redland_example
+- config/talia_core.yml
+- config/talia_core.yml.example
+- generators/generator_helpers.rb
+- generators/talia_admin/USAGE
+- generators/talia_admin/talia_admin_generator.rb
+- generators/talia_admin/templates/controllers/admin/background_controller.rb
+- generators/talia_admin/templates/controllers/admin/custom_templates_controller.rb
+- generators/talia_admin/templates/controllers/admin/locales_controller.rb
+- generators/talia_admin/templates/controllers/admin/sources_controller.rb
+- generators/talia_admin/templates/controllers/admin/translations_controller.rb
+- generators/talia_admin/templates/controllers/admin/users_controller.rb
+- generators/talia_admin/templates/controllers/admin_controller.rb
+- generators/talia_admin/templates/helpers/admin/background_helper.rb
+- generators/talia_admin/templates/helpers/admin/custom_templates_helper.rb
+- generators/talia_admin/templates/helpers/admin/locales_helper.rb
+- generators/talia_admin/templates/helpers/admin/sources_helper.rb
+- generators/talia_admin/templates/helpers/admin/translations_helper.rb
+- generators/talia_admin/templates/helpers/admin/users_helper.rb
+- generators/talia_admin/templates/helpers/admin_helper.rb
+- generators/talia_admin/templates/public/images/backend/body_bg.gif
+- generators/talia_admin/templates/public/images/backend/footer_bg.gif
+- generators/talia_admin/templates/public/images/backend/header.jpg
+- generators/talia_admin/templates/public/images/backend/header_bg.gif
+- generators/talia_admin/templates/public/images/backend/menu.jpg
+- generators/talia_admin/templates/public/images/backend/menu_bg.gif
+- generators/talia_admin/templates/public/images/backend/opednid.gif
+- generators/talia_admin/templates/public/images/backend/page_bg.jpg
+- generators/talia_admin/templates/public/images/backend/triangolino.gif
+- generators/talia_admin/templates/public/images/backend/triangolino_full.gif
+- generators/talia_admin/templates/public/javascripts/backend.js
+- generators/talia_admin/templates/public/javascripts/lowpro.js
+- generators/talia_admin/templates/public/stylesheets/backend.css
+- generators/talia_admin/templates/test/fixtures/users.yml
+- generators/talia_admin/templates/test/functional/admin/custom_templates_controller_test.rb
+- generators/talia_admin/templates/test/functional/admin/locales_controller_test.rb
+- generators/talia_admin/templates/test/functional/admin/sources_controller_test.rb
+- generators/talia_admin/templates/test/functional/admin/translations_controller_test.rb
+- generators/talia_admin/templates/test/functional/admin/users_controller_test.rb
+- generators/talia_admin/templates/test/functional/admin_controller_test.rb
+- generators/talia_admin/templates/test/unit/user_test.rb
+- generators/talia_admin/templates/views/admin/background/_finished.html.erb
+- generators/talia_admin/templates/views/admin/background/_pending.html.erb
+- generators/talia_admin/templates/views/admin/background/_progress.html.erb
+- generators/talia_admin/templates/views/admin/background/_running.html.erb
+- generators/talia_admin/templates/views/admin/background/environment.html.erb
+- generators/talia_admin/templates/views/admin/background/show.html.erb
+- generators/talia_admin/templates/views/admin/background/stderr.html.erb
+- generators/talia_admin/templates/views/admin/background/stdin.html.erb
+- generators/talia_admin/templates/views/admin/background/stdout.html.erb
+- generators/talia_admin/templates/views/admin/custom_templates/_content_form_column.rhtml
+- generators/talia_admin/templates/views/admin/custom_templates/_template_type_form_column.rhtml
+- generators/talia_admin/templates/views/admin/index.html.erb
+- generators/talia_admin/templates/views/admin/locales/new.html.erb
+- generators/talia_admin/templates/views/admin/sources/_data.html.erb
+- generators/talia_admin/templates/views/admin/sources/_data_form.html.erb
+- generators/talia_admin/templates/views/admin/sources/_form.html.erb
+- generators/talia_admin/templates/views/admin/sources/_list.html.erb
+- generators/talia_admin/templates/views/admin/sources/_notice.html.erb
+- generators/talia_admin/templates/views/admin/sources/_predicate.html.erb
+- generators/talia_admin/templates/views/admin/sources/_predicates.html.erb
+- generators/talia_admin/templates/views/admin/sources/_sources.html.erb
+- generators/talia_admin/templates/views/admin/sources/_upload.html.erb
+- generators/talia_admin/templates/views/admin/sources/edit.html.erb
+- generators/talia_admin/templates/views/admin/sources/index.html.erb
+- generators/talia_admin/templates/views/admin/translations/_new_translation.html.erb
+- generators/talia_admin/templates/views/admin/translations/_translation.html.erb
+- generators/talia_admin/templates/views/admin/translations/edit.html.erb
+- generators/talia_admin/templates/views/admin/users/_form.html.erb
+- generators/talia_admin/templates/views/admin/users/_form_roles.html.erb
+- generators/talia_admin/templates/views/admin/users/edit.html.erb
+- generators/talia_admin/templates/views/admin/users/index.html.erb
+- generators/talia_admin/templates/views/admin/users/new.html.erb
+- generators/talia_admin/templates/views/admin/users/show.html.erb
+- generators/talia_admin/templates/views/layouts/admin.html.erb
+- generators/talia_base/USAGE
+- generators/talia_base/talia_base_generator.rb
+- generators/talia_base/templates/app/controllers/custom_templates_controller.rb
+- generators/talia_base/templates/app/controllers/ontologies_controller.rb
+- generators/talia_base/templates/app/controllers/sessions_controller.rb
+- generators/talia_base/templates/app/controllers/source_data/show.html.erb
+- generators/talia_base/templates/app/controllers/source_data_controller.rb
+- generators/talia_base/templates/app/controllers/sources_controller.rb
+- generators/talia_base/templates/app/controllers/types_controller.rb
+- generators/talia_base/templates/app/helpers/custom_templates_helper.rb
+- generators/talia_base/templates/app/helpers/ontologies_helper.rb
+- generators/talia_base/templates/app/helpers/sessions_helper.rb
+- generators/talia_base/templates/app/helpers/source_data/show.html.erb
+- generators/talia_base/templates/app/helpers/source_data_helper.rb
+- generators/talia_base/templates/app/helpers/sources_helper.rb
+- generators/talia_base/templates/app/helpers/types_helper.rb
+- generators/talia_base/templates/app/views/ontologies/index.builder
+- generators/talia_base/templates/app/views/ontologies/show.builder
+- generators/talia_base/templates/app/views/sessions/new.html.erb
+- generators/talia_base/templates/app/views/source_data/show.html.erb
+- generators/talia_base/templates/app/views/sources/_form.html.erb
+- generators/talia_base/templates/app/views/sources/edit.html.erb
+- generators/talia_base/templates/app/views/sources/index.html.erb
+- generators/talia_base/templates/app/views/sources/new.html.erb
+- generators/talia_base/templates/app/views/sources/show.html.erb
+- generators/talia_base/templates/app/views/types/index.html.erb
+- generators/talia_base/templates/app/views/types/show.html.erb
+- generators/talia_base/templates/config/routes.rb
+- generators/talia_base/templates/config/talia_initializer.rb
+- generators/talia_base/templates/migrations/bj_migration.rb
+- generators/talia_base/templates/migrations/constraint_migration.rb
+- generators/talia_base/templates/migrations/create_active_sources.rb
+- generators/talia_base/templates/migrations/create_custom_templates.rb
+- generators/talia_base/templates/migrations/create_data_records.rb
+- generators/talia_base/templates/migrations/create_open_id.rb
+- generators/talia_base/templates/migrations/create_progress_jobs.rb
+- generators/talia_base/templates/migrations/create_roles.rb
+- generators/talia_base/templates/migrations/create_semantic_properties.rb
+- generators/talia_base/templates/migrations/create_semantic_relations.rb
+- generators/talia_base/templates/migrations/create_sessions.rb
+- generators/talia_base/templates/migrations/create_users.rb
+- generators/talia_base/templates/migrations/create_workflows.rb
+- generators/talia_base/templates/migrations/populate_users.rb
+- generators/talia_base/templates/migrations/upgrade_relations.rb
+- generators/talia_base/templates/ontologies/hyper_ontology.owl
+- generators/talia_base/templates/ontologies/hyper_ontology.pprj
+- generators/talia_base/templates/ontologies/hyper_ontology.repository
+- generators/talia_base/templates/ontologies/scholar_0.1.owl
+- generators/talia_base/templates/script/configure_talia
+- generators/talia_base/templates/script/prepare_images
+- generators/talia_base/templates/talia.sh
+- generators/talia_base/templates/tasks/talia_core.rk
 - lib/JXslt/jxslt.rb
 - lib/acts_as_roled.rb
 - lib/core_ext.rb
@@ -92,6 +225,7 @@ files:
 - lib/talia_util/io_helper.rb
 - lib/talia_util/progressable.rb
 - lib/talia_util/progressbar.rb
+- lib/talia_util/rake_tasks.rb
 - lib/talia_util/rdf_update.rb
 - lib/talia_util/some_sigla.xml
 - lib/talia_util/test_helpers.rb
@@ -99,6 +233,7 @@ files:
 - lib/talia_util/yaml_import.rb
 - lib/user.rb
 - lib/version.rb
+- tasks/talia_core_tasks.rake
 - README.rdoc
 rubygems_version: 1.3.5
 rdoc_options: 
@@ -226,7 +361,7 @@ dependencies:
     requirements: 
     - - '>='
       - !ruby/object:Gem::Version 
-        version: 0.8.1
+        version: 0.6.1
 - !ruby/object:Gem::Dependency 
   type: :runtime
   name: oai
@@ -277,16 +412,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version 
         version: 0.7.1
-- !ruby/object:Gem::Dependency 
-  type: :runtime
-  name: json
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    version: 
-    requirements: 
-    - - '>='
-      - !ruby/object:Gem::Version 
-        version: 1.1.0
 description: This is the core plugin for building a digital library with Talia/Rails.
 email: ghub@limitedcreativity.org
 authors: